diff --git a/dune-project b/dune-project index 400dbcf4457ce275d2597441c3af249956801f39..57dc3a6697108ce0d01ad4e4aea38607a5ca5350 100644 --- a/dune-project +++ b/dune-project @@ -153,7 +153,6 @@ (package (name tezos-tree-encoding-test)(allow_empty)) (package (name tezos_internal_brassaia_tests)(allow_empty)) (package (name tezos_internal_irmin_tests)(allow_empty)) -(package (name tezt-cloud)(allow_empty)) (package (name tezt-etherlink)(allow_empty)) (package (name tezt-riscv-sandbox)(allow_empty)) (package (name tezt-tests-cloud)(allow_empty)) diff --git a/etherlink/tezt/lib/dune b/etherlink/tezt/lib/dune index 2c4a0631ac5cb1ff96903e1f8559df7aa3e5ce70..117cde38f96ef925593fe3bf1d814832bd75e078 100644 --- a/etherlink/tezt/lib/dune +++ b/etherlink/tezt/lib/dune @@ -8,11 +8,13 @@ octez-libs.tezt-wrapper tezt-tezos.tezt-performance-regression octez-libs.crypto - tezt-tezos) + tezt-tezos + tezt-tezos.tezt-cloud) (flags (:standard) -open Tezt_wrapper -open Tezt_wrapper.Base -open Tezt_tezos_tezt_performance_regression -open Tezt_tezos - -open Tezt_tezos.Runnable.Syntax)) + -open Tezt_tezos.Runnable.Syntax + -open Tezt_tezos_tezt_cloud)) diff --git a/etherlink/tezt/lib/evm_node.mli b/etherlink/tezt/lib/evm_node.mli index d60096cf2fe990bba08af2b1acf0e6de7189ceb9..d707e23adfab2f5b2063878c3edbe6709d63aeb8 100644 --- a/etherlink/tezt/lib/evm_node.mli +++ b/etherlink/tezt/lib/evm_node.mli @@ -416,7 +416,7 @@ module Agent : sig ?data_dir:string -> ?mode:mode -> string -> - Tezt_tezos.Agent.t -> + Agent.t -> t Lwt.t val init : @@ -425,7 +425,7 @@ module Agent : sig ?mode:mode -> ?data_dir:string -> string -> - Tezt_tezos.Agent.t -> + Agent.t -> t Lwt.t val rpc_port : t -> int diff --git a/manifest/product_etherlink.ml b/manifest/product_etherlink.ml index 81f9058eb4e3f3581f688839a59c9ad558277a77..a6d48cceb2dc1efd0992ef0c90f1cf5fde960ac3 100644 --- a/manifest/product_etherlink.ml +++ b/manifest/product_etherlink.ml @@ -31,6 +31,7 @@ let tezt_etherlink = tezt_performance_regression |> open_; octez_crypto; tezt_tezos |> open_ |> open_ ~m:"Runnable.Syntax"; + tezt_cloud |> open_; ] ~release_status:Unreleased diff --git a/manifest/product_octez.ml b/manifest/product_octez.ml index af5c33d81e105512ccc642ead370ab0168979ec0..7e1d0c23bddf58a312e0367f32566228068d921d 100644 --- a/manifest/product_octez.ml +++ b/manifest/product_octez.ml @@ -2316,6 +2316,19 @@ let tezt_performance_regression = ~bisect_ppx:No ~deps:[tezt_wrapper |> open_ |> open_ ~m:"Base"; uri; cohttp_lwt_unix] +let tezt_cloud = + public_lib + "tezt-tezos.tezt-cloud" + ~path:"tezt/lib_cloud" + ~opam:"tezt-tezos" + ~bisect_ppx:No + ~deps: + [ + tezt_lib |> open_ |> open_ ~m:"Base"; + tezt_performance_regression |> open_; + data_encoding; + ] + let tezt_tezos = public_lib "tezt-tezos" @@ -2327,6 +2340,7 @@ let tezt_tezos = [ tezt_wrapper |> open_ |> open_ ~m:"Base"; tezt_performance_regression |> open_; + tezt_cloud |> open_; uri; hex; octez_crypto_dal; @@ -2337,23 +2351,6 @@ let tezt_tezos = ~cram:true ~release_status:Released -let tezt_cloud = - private_lib - "tezt_cloud" - ~path:"tezt/lib_cloud" - ~opam:"tezt-cloud" - ~synopsis: - "A library to ease the writing of test using machines onto cloud \ - providers (GCP, AWS, ...)" - ~bisect_ppx:No - ~deps: - [ - tezt_lib |> open_ |> open_ ~m:"Base"; - tezt_tezos |> open_ |> open_ ~m:"Runnable.Syntax"; - tezt_performance_regression |> open_; - ] - ~release_status:Unreleased - let octez_p2p_test_common = octez_shell_lib "p2p_test_common" diff --git a/opam/tezt-cloud.opam b/opam/tezt-cloud.opam deleted file mode 100644 index d592297b88ba306dbf99010ab0276ad031912cea..0000000000000000000000000000000000000000 --- a/opam/tezt-cloud.opam +++ /dev/null @@ -1,21 +0,0 @@ -# This file was automatically generated, do not edit. -# Edit file manifest/main.ml instead. -opam-version: "2.0" -maintainer: "contact@tezos.com" -authors: ["Tezos devteam"] -homepage: "https://www.tezos.com/" -bug-reports: "https://gitlab.com/tezos/tezos/issues" -dev-repo: "git+https://gitlab.com/tezos/tezos.git" -license: "MIT" -depends: [ - "dune" { >= "3.11.1" } - "ocaml" { >= "4.14" } - "tezt" { >= "4.1.0" & < "5.0.0" } - "tezt-tezos" { = version } -] -build: [ - ["rm" "-r" "vendors" "contrib"] - ["dune" "build" "-p" name "-j" jobs] - ["dune" "runtest" "-p" name "-j" jobs] {with-test} -] -synopsis: "A library to ease the writing of test using machines onto cloud providers (GCP, AWS, ...)" diff --git a/opam/tezt-tests-cloud.opam b/opam/tezt-tests-cloud.opam index e51cb0446a43e63d3d3e6d9eeff30c4e03ef7aa1..3e67a6b299a5f32e3f8764e498b3ea88fe2e326c 100644 --- a/opam/tezt-tests-cloud.opam +++ b/opam/tezt-tests-cloud.opam @@ -12,7 +12,6 @@ depends: [ "ocaml" { >= "4.14" } "octez-libs" "tezt-tezos" - "tezt-cloud" "tezt-etherlink" { = version } ] build: [ diff --git a/opam/tezt-tezos.opam b/opam/tezt-tezos.opam index aac3c416373ff665ab75f7af150526b4dedb4f23..bd2d08e5ad21ecf8f18dfbd3c0caaf8fd67ed21e 100644 --- a/opam/tezt-tezos.opam +++ b/opam/tezt-tezos.opam @@ -13,8 +13,8 @@ depends: [ "octez-libs" { = version } "uri" { >= "3.1.0" } "cohttp-lwt-unix" { >= "5.2.0" } + "tezt" { >= "4.1.0" & < "5.0.0" } "hex" { >= "1.3.0" } - "tezt" { with-test & >= "4.1.0" & < "5.0.0" } ] build: [ ["rm" "-r" "vendors" "contrib"] diff --git a/tezt/lib_cloud/README.md b/tezt/lib_cloud/README.md index 2dc82486dc3adf3e84189b678d589a3fb9711378..fc21a44975a438847722a92959ef2f0ebf253fe9 100644 --- a/tezt/lib_cloud/README.md +++ b/tezt/lib_cloud/README.md @@ -1,11 +1,20 @@ # Tezt cloud -This library can used to write Tezt tests that can use machines deployed in the -cloud. At the moment those machines must be deployed onto GCP, and hence require -a GCP account to be used. +This library can be used to write Tezt tests that can use VMs deployed in the +cloud. At the moment those VMs must be deployed onto GCP, and hence require a +GCP account to be used. Scenarios written with tezt cloud can also be run in +localhost without using GCP. # Initialisation +There are two important configuration steps before using the library: + +- Having a GCP account +- Have your own dockerfile (or choose an existent one that suits you) + +Do note that the GCP account is not necessary if you use the library +in `localhost` mode (and so you don't need `gcloud` and `terraform`). + ## Creating a GCP account Check with your manager how you can have a GCP account and to which @@ -42,7 +51,7 @@ Run the following commands: with your login. 4. `gcloud services enable artifactregistry.googleapis.com`: Allow using the - artifact registries of Google (useful for pushing docker images). + artifact registries of Google (necessary for pushing docker images). 5. `sudo usermod -aG docker $USER` where `$USER` is your user. This is to enable `docker` commands to be run without being sudo. @@ -50,22 +59,47 @@ Run the following commands: 6. `gcloud config set project [YOUR_PROJECT_ID]`: For configuring the default project. -7. (optional) `gcloud auth configure-docker europe-west1-docker.pkg.dev`: - Authorize the artifact registry used by this project. +7. (optional) `gcloud auth configure-docker + europe-west1-docker.pkg.dev`: Authorize the artifact registry used + by this project. This can be useful if you plan to push docker + images without tezt cloud. + +# Tezt cloud variable -Please set an environment variable `TEZT_CLOUD` which will be used to identify -the resources you deploy onto the cloud. The value should be unique for the -given project. We recommend using a similar username as the one you use on -Gitlab. +Any time you use the Tezt cloud library, a namespace for your +experiment must be provided. Most of the time, if you only run one +experiment at a time, you can use your gitlab username for example. +Such a namespace can be provided via the environment variable +`TEZT_CLOUD` or via the CLI option `--tezt-cloud`. It may be +interesting to set the `TEZT_CLOUD` environment variable with your +gitlab username since this is the one you will use the most. + +This variable is also used to determine the dockerfile to be used for +your experiment. In case you want to use a different dockerfile, you +can use the CLI option `--dockerfile-alias`. # Docker -The library assumes that you have a dockerfile image in the subdirectory of the -library `tezt/lib_cloud/dockerfiles` (it can be committed so that it gives -people an easy way to copy/paste the dockerfile for they own purpose). This -image will be the one that will be installed on each running VM. The library -takes care for you to push this image on a dedicated GCP registry by running the -appropriate command (see below). +The library relies mainly on docker. Each VM that will be deployed +will run a docker image. This docker image must follow some conditions +(mainly running an SSH server), but apart from that you are free to +use your own dockerfile. + +To help you write such an image, a default one based on debian is +provided in `tezt/lib_cloud/saroupille.Dockerfile`. + +The library takes care for you to push this image on a dedicated GCP +registry. A docker image such as the one generated from +`saroupille.Dockerfile` may contain some binaries. Tezt cloud always +try to rebuild the docker image to be sure it uses the last version, +however be sure to compile the binaries beforehand. + +Finally, do note that with Tezt cloud you can specify for each VM +individually the docker image to be used. The one provided by the +dockerfile as discussed above will be the one by default. It is also +possible to use docker images published by the Octez CI. At the +moment, the docker image from the latest released of Octez is +supported. ## Write your own dockerfile @@ -73,44 +107,29 @@ Depending on your local setup, we recommend to copy/paste an image that is similar to your local setup. An example for a ubuntu/debian-like machine is given by `saroupille.Dockerfile`. At the moment, several assumptions are made: -1. An ssh server is running and you have access to it. -2. The ssh key should be in the default ssh directory and named - `$(TF_WORKSPACE)-tf`. The docker variable for specifying the ssh key should - be named `SSH_PUBLIC_KEY`. -3. For convenience, we recommend the image to contain the minimum necessary to +1. An ssh server is running on port `30000` +2. For convenience, we recommend the image to contain the minimum necessary to run Octez binaries that were compiled on your local setup. In particular, we use the docker variable `ZCASH_PARAMS_PATH` to provide the place where the zcash parameters are used. -4. Optionally, you can also put the binaries into the docker image (see +3. Optionally, you can also put the binaries into the docker image (see `saroupille.Dockerfile`). -# One-time deployment - -The library `tezt-cloud` needs to deploy resources that will be used throughout -your different experimentations and these resources should not be destroyed. - -At the moment there are two resources of this kind which are detailed -below. +To ensure you can connect to the ssh server, the library generates (if +it does not exist) and ssh key based on the `tezt-cloud` variable. -## Terraform state - -Terraform maintains a state of the deployed resources that are *owned* by -`terraform`. A standard way to store them is to use a GCP bucket. +# One-time deployment -Once your `TEZT_CLOUD` value is set and you have the appropriate -rights, you should be able to create the bucket using: - -```bash -dune exec tezt/tests/cloud/main.exe -- terraform state deploy bucket -v -``` +The library `tezt-cloud` needs to deploy resources that will be used +and reused throughout your different experimentations. Tezt cloud +should deploy them for you automatically. Hence the commands in this +section are optional. ## Docker registry -A docker registry is needed so that, during initialisation of the VMs, -the docker image can be pulled from the cloud. - -Once your `TEZT_CLOUD` value is set and you have the appropriate -rights, you should be able to create the registry using: +Tezt cloud does a best effort to create a docker registry if it does +not exists already. The name of the docker registry depends on the +`TEZT_CLOUD` value. ```bash dune exec tezt/tests/cloud/main.exe -- docker registry deploy -v @@ -118,7 +137,7 @@ dune exec tezt/tests/cloud/main.exe -- docker registry deploy -v ## Deployment -To push your image, you should run: +If you want to push your image manually on the registry you can do: ```bash dune exec tezt/tests/cloud/main.exe -- docker push -v @@ -126,12 +145,17 @@ dune exec tezt/tests/cloud/main.exe -- docker push -v ## Run your first scenario -Any scenario can be run on the local host for checking any deployment -error. Hence, when testing any scenario we recommend to provide the -`--localhost` parameter with a low number of machines to check whether your -scenario works as expected. The number of VMs can be specified on the -command-line via `--vms`. But it can also be computed by the scenario, so that -`--vms` is not mandatory. +Any Tezt cloud scenario can be run on the host machine for checking +any deployment error, this is the `localhost` mode. Hence, when +testing any scenario we recommend to provide the `--localhost` +parameter with a low number of machines to check whether your scenario +works as expected. + +When you are ready to deploy resources, you can remove the +`--localhost` option, or alternatively use the `--cloud` option. When +deploying machines on the cloud, the number of VMs can be specified on +the command-line via `--vms`. But it can also be computed by the +scenario, so that `--vms` is not mandatory. We recommend to test it twice: using localhost or not: @@ -148,7 +172,7 @@ dune exec tezt/tests/cloud/main.exe -- cloud health -v (Any scenario using this library should contain the tag `cloud`.) If these two tests succeed, you are ready to create your own scenarios with -`tezt-cloud`. +`tezt-cloud` or use existent ones. ## Destroying VMs @@ -162,15 +186,16 @@ running: dune exec tezt/tests/cloud/main.exe -- terraform destroy -v ``` -(It is actually quite difficult to provide an automatic way to destroy resources -published onto the cloud. A possible solution in the future would be to deploy a -cron job locally.) +However, all your VMs may come with a time to live parameter. For the +`--cloud` mode, it is set to 2 hours. For the proxy mode (see below), +it is not set by default. ## Monitoring Tezt-cloud offers you a way to monitor your VMs via [netdata](https://www.netdata.cloud/). To enable monitoring, you only have to provide the option `--monitoring` while running your scenario. +In which case, all the VMs will be monitored. The netdata dashboard can be accessed on port `19999` on the remote machine. Do note that if you use localhost, the monitoring will monitor your entire machine. @@ -184,45 +209,132 @@ stopped until the VM is being teared down. ## Website -A website can be run to summarize information about the experiment. This can be -activated with the option `--website`. The website runs at -`http://localhost:8080`. The port can be changed with option `--website-port`. -The website can also be used to report ad-hoc metrics (see below). +A website can be run to summarize information about the experiment. +This can be activated with the option `--website`. The website runs at +`http://localhost:8080`. The port can be changed with option +`--website-port`. The website can also be used to report ad-hoc +metrics (see in the next section). ## Prometheus -Prometheus can be used to export metrics from `netdata`. A prometheus container -can be spawned on the host machine via the option `--prometheus`. As a -side-effect, this will also export a snapshot at the end of the experiment. By -default, the snapshot will be stored in the `/tmp` directory. The directory can -be modified with the `--prometheus-snapshot-directory` option. The name of the -snapshot can be set via `--prometheus-snapshot`. +Prometheus can be used to export metrics from `netdata` or export +Octez metrics. A prometheus container can be spawned on the host +machine via the option `--prometheus`. As a side-effect, this will +also export a snapshot at the end of the experiment. By default, the +snapshot will be stored in the `/tmp` directory. The name of the +snapshot can be specified via `--prometheus-snapshot-filename`. After being exported, a snapshot can be imported by running: ```bash -dune exec tezt/tests/cloud/main.exe -- prometheus import --prometheus-snapshot +dune exec tezt/tests/cloud/main.exe -- prometheus import --prometheus-snapshot-filename ``` -## Exporting ad-hoc metrics +### Exporting ad-hoc metrics -While writing an experiment, it may be interesting to export metrics. This is -doable if the experiment is run with options `--website` and `--prometheus`. By -doing so, metrics can be exported via the function `Cloud.push_metric`. It takes -as parameters the name of the metric as well as optional labels and the value to -export. Metrics are reported on a special page `/metrics.txt` on the website and -will be scrapped by the Prometheus instance. Such metrics will be automatically -snapshotted at the end of the experiments. +While writing an experiment, it may be interesting to export metrics +to the scenario run (for example to ensure the scenario behaves as +expected). This feature requires the experiment to be run with +`--website` and `--prometheus` (which is the case by default). By +doing so, metrics can be exported via the function +`Cloud.push_metric`. It takes as parameters the name of the metric as +well as optional labels and the value to export. Metrics are provided +on the website at the following URI `/metrics.txt` and will be +scrapped by the Prometheus instance. Those metrics are also exported +at the end of the experiment in the Prometheus snapshot. -## Exporting daemons metrics +### Exporting daemons metrics -Metrics exported by daemons such as `octez-node` or `octez-dal-node` can also be -exported to the Prometheus instance using the function +When writing a new scenario, you can instruct how to export daemons +(e.g. `octez-node`) metrics to Prometheus by calling the function `Cloud.add_prometheus_source`. +## Grafana + +Grafana can be used to visualize prometheus metrics (activated by +default). This can be accessed on `http://localhost:3000`. Tezt cloud +comes with a set of dashboards so that it is ready to use. Feel free +to update/add new dashboards in +`tezt/lib_cloud/grafana/dasboards.json`. + +## Proxy mode + +**EXPERIMENTAL** + +The `--proxy` option allow you to run your experiment onto a VM +instead of your local machine. This may have several benefits, but in +particular, it can be used to set up long experiments (that can last +for weeks). + +When running a test in `--proxy` mode, you can detach it after the +deployement has been made and the scenario start to kick in. In that +case, you can detach from your experiment by pressing `ctrl+d`. By +doing so, you can do whatever you want, but the experiment is still +executed on a remote VM. + +To reattach the test, simply rerun the command. Normally, it should +reattach back the running experiment. + +Killing an experiment manually requires some care. You must do it in +two steps: + +1. Press Ctrl+C. This will trigger the shutdown of the +experiment. + +2. Once you see the scenario has ended properly, you can press Ctrl+D. + +If you make a mistake during the shutdown, the scenario may not end +properly. In that case, you can clean up the VMs by running: + +``` +dune exec tezt/tests/cloud/main.exe -- cloud clean up -v +``` + +UX-wise, the proxy mode has still some awkwards behaviour regarding +how log events are printed on `Ctrl+D` or `Ctrl+C`. Moreover, the +handling of `sigint` (Ctrl+C) may be a bit fragile. + +We recommend to use first the `Cloud` mode before using the `Proxy` +mode to ensure the scenario behaves as expected. + +With the proxy mode, we recommend to use the website if you want to +follow your experiment. The website should be provided in the logs +once you detach from the experiment. + +Otherwise, you can connect to it by looking at the external IP address +of the `proxy` machine on GCP and use the port `8080`. + +### DNS + +The website deployed with the proxy mode can be associated with a +domain name if the `--dns` argument is provided (true by default). + +The domain associated with the website will be: +`http://..`. + +To make it work, any project must register a subdomain zone +beforehand. This can be done via: + +``` +dune exec tezt/tests/cloud/main.exe -- cloud create dns zone -v --dns-domain +``` + +The name servers associated with this domain must be added manually to +the domain regstriy associated with ``. + +You can check this work as expected by running: + +``` +dig ns . +``` + +You should see the name servers you just added. + + ## Recommendation when writing experiments -0. Never forget to destroy your VMs. +0. Even though there is a time to live for the VM, please don't forget + to destroy your VMs. 1. You can spawn mutiple daemons on the same VM. @@ -233,9 +345,11 @@ exported to the Prometheus instance using the function use `Agent.next_port`. 3. The runner that will be used (this can be given to the agent). -3. Whenever it is possible, write your test without depending specifically on a - fixed number of machines. Whenver you want to increase or decrease the number - of machines, you can just run your test again and it will just work. +3. Whenever it is possible, write your test without depending + specifically on a fixed number of machines. Whenver you want to + increase or decrease the number of machines, you can just run your + test again and it will just work. This can be done by using a + dispenser (via the OCaml module `Seq` for example). 4. Use `--log-file` option from Tezt so that you can inspect errors a posteriori. @@ -305,21 +419,3 @@ exported to the Prometheus instance using the function ControlPath ~/.ssh/master-%r@%h:%p ControlPersist 120 ``` - -## Limitations and future work - -At the moment, there are two main limitations to `tezt-cloud`: - -- The host machine receives and parses all the events fromm all the daemons. It - could be interesting instead to have a `proxy` mode so that intermediate - processes are hosted on VMs themselves to catch some of those events. This - could be also used as a basis to implement `detach/attach` capabilities to - Tezt cloud so that you don't have to be connected all the time on your machine - during the experiment. - -- All the VMs must use the same kind of machine. For performance-related - experiments, it may be interesting to have different kinds of machines for - different kinds of VMs. While this could be supported in future versions of - `tezt-cloud`, the only way to by-pass this limitation is to use multiple - `workspace`s (you need to implement the logic to handle multiple workspaces - yourself). diff --git a/tezt/lib_cloud/agent.ml b/tezt/lib_cloud/agent.ml new file mode 100644 index 0000000000000000000000000000000000000000..f053ff68e3fcb9d641bc643533f569bf173fe16b --- /dev/null +++ b/tezt/lib_cloud/agent.ml @@ -0,0 +1,242 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +type t = { + (* The name initially is the same as [vm_name] and can be changed dynamically by the scenario. *) + mutable name : string; + vm_name : string; + zone : string option; + point : string * int; + runner : Runner.t; + next_available_port : unit -> int; + configuration : Configuration.t; +} + +let ssh_id () = + if Env.mode = `Orchestrator then Env.ssh_private_key_filename ~home:"/root" () + else Env.ssh_private_key_filename () + +let docker_image_encoding = + let open Data_encoding in + union + [ + case + ~title:"gcp" + Json_only + string + (function Env.Gcp {alias} -> Some alias | _ -> None) + (fun alias -> Gcp {alias}); + case + ~title:"octez_latest_release" + Json_only + (constant "octez-latest-release") + (function Env.Octez_latest_release -> Some () | _ -> None) + (fun () -> Octez_latest_release); + ] + +let configuration_encoding = + let open Data_encoding in + let open Configuration in + conv + (fun {machine_type; binaries_path; docker_image; max_run_duration = _} -> + (machine_type, binaries_path, docker_image)) + (fun (machine_type, binaries_path, docker_image) -> + Configuration.make ~machine_type ~binaries_path ~docker_image ()) + (obj3 + (req "machine_type" Data_encoding.string) + (req "binaries_path" Data_encoding.string) + (req "docker_image" docker_image_encoding)) + +let encoding = + let open Data_encoding in + conv + (fun { + name; + vm_name; + zone; + point; + runner = _; + next_available_port; + configuration; + } -> + (name, vm_name, zone, point, next_available_port (), configuration)) + (fun (name, vm_name, zone, point, next_available_port, configuration) -> + let ssh_port = snd point in + let address = fst point in + let ssh_id = ssh_id () in + let runner = + Runner.create ~ssh_user:"root" ~ssh_id ~ssh_port ~address () + in + let next_available_port = + let current_port = ref (next_available_port - 1) in + fun () -> + incr current_port ; + !current_port + in + {name; vm_name; zone; point; runner; next_available_port; configuration}) + (obj6 + (req "name" Data_encoding.string) + (req "vm_name" Data_encoding.string) + (req "zone" (Data_encoding.option Data_encoding.string)) + (req + "point" + (Data_encoding.tup2 Data_encoding.string Data_encoding.int31)) + (req "next_available_port" Data_encoding.int31) + (req "configuration" configuration_encoding)) + +let make ?zone ~ssh_id ~point:((address, ssh_port) as point) ~configuration + ~next_available_port ~name () = + let ssh_user = "root" in + let runner = Runner.create ~ssh_user ~ssh_id ~ssh_port ~address () in + { + point; + runner; + name; + vm_name = name; + next_available_port; + configuration; + zone; + } + +let name {name; _} = name + +let vm_name {vm_name; _} = vm_name + +let point {point; _} = point + +let cmd_wrapper {zone; vm_name; _} = + match zone with + | None -> None + | Some zone -> + let ssh_private_key_filename = + if Env.mode = `Orchestrator then + Env.ssh_private_key_filename ~home:"/root" () + else Env.ssh_private_key_filename () + in + Some (Gcloud.cmd_wrapper ~zone ~vm_name ~ssh_private_key_filename) + +let set_name agent name = agent.name <- name + +let path_of agent binary = agent.configuration.binaries_path // binary + +let host_run_command agent cmd args = + match cmd_wrapper agent with + | None -> Process.spawn cmd args + | Some cmd_wrapper -> + Process.spawn cmd_wrapper.Gcloud.cmd (cmd_wrapper.args @ [cmd] @ args) + +let docker_run_command agent cmd args = + let cmd, args = + Runner.wrap_with_ssh agent.runner (Runner.Shell.cmd [] cmd args) + in + (* The host check is very not convenient at all for this setting. This makes + tezt cloud sensible to man in the middle attacks. There are other options + like doing the check the first time only etc... But they all fail at some + point. I think the issue is that: + + - GCP may reuse IP addresses + - The docker image generates new key anytime it is generated + + I don't have a good proposition that keeps a nice UX and is secure at the moment. + *) + Process.spawn cmd (["-o"; "StrictHostKeyChecking=no"] @ args) + +let copy agent ~source ~destination = + let runner = agent.runner in + let identity = + Option.fold ~none:[] ~some:(fun i -> ["-i"; i]) runner.ssh_id + in + let port = + Option.fold + ~none:[] + ~some:(fun p -> ["-P"; Format.sprintf "%d" p]) + runner.ssh_port + in + let* exists = + let process = Process.spawn ~runner "ls" [destination] in + let* status = process |> Process.wait in + match status with + | WEXITED 0 -> + let hash_of_md5_output output = + String.trim output |> String.split_on_char ' ' |> List.hd + in + (* If the file already exists on the remote machine, we compare the + hashes to be sure they are the same. *) + Lwt.catch + (fun () -> + let* destination_hash = + let* output = + docker_run_command agent "md5sum" [destination] + |> Process.check_and_read_stdout + in + (* md5sum output is: ' '. We only take the hash. *) + Lwt.return (hash_of_md5_output output) + in + let process = Process.spawn "md5sum" [source] in + let* status = process |> Process.wait in + match status with + | WEXITED 0 -> + let* output = Process.check_and_read_stdout process in + let source_hash = hash_of_md5_output output in + Lwt.return (destination_hash = source_hash) + | _ -> Lwt.return_true) + (fun _ -> Lwt.return_false) + | _ -> Lwt.return_false + in + if exists then Lwt.return_unit + else + let destination = + Format.sprintf + "%s%s:%s" + (Option.fold + ~none:"" + ~some:(fun u -> Format.sprintf "%s@" u) + runner.ssh_user) + runner.address + destination + in + let* () = + (* FIXME: I forgot why we enforce [-0]. *) + Process.run + "scp" + (["-O"] + @ ["-o"; "StrictHostKeyChecking=no"] + @ identity @ port @ [source] @ [destination]) + in + Lwt.return_unit + +let copy = + (* We memoize the copy so that it is done at most once per destination per + scenario. This optimisation ease the writing of scenario so that copy can + always be called before using the file copied. *) + let already_copied = Hashtbl.create 11 in + fun ?destination agent ~source -> + let destination = + Option.value ~default:(path_of agent source) destination + in + match Hashtbl.find_opt already_copied (agent, destination) with + | Some promise -> promise + | None -> + let p = + let* () = + docker_run_command + agent + "mkdir" + ["-p"; Filename.dirname destination] + |> Process.check + in + let* () = copy agent ~source ~destination in + Lwt.return destination + in + Hashtbl.replace already_copied (agent, destination) p ; + p + +let next_available_port t = t.next_available_port () + +let runner {runner; _} = runner + +let configuration {configuration; _} = configuration diff --git a/tezt/lib_tezos/agent.mli b/tezt/lib_cloud/agent.mli similarity index 63% rename from tezt/lib_tezos/agent.mli rename to tezt/lib_cloud/agent.mli index 816e615a977bd28f067c9f5caf996d57d5cb2530..5c545ed7a2418150b6c86a62656d18ae2209be69 100644 --- a/tezt/lib_tezos/agent.mli +++ b/tezt/lib_cloud/agent.mli @@ -10,33 +10,39 @@ (** Datatype for an agent *) type t -(** [make ?binaries_path ?ssh_user ~ssh_id ~point ~next_available_port ~name] +(** [make ?binaries_path ~ssh_id ~point ~next_available_port ~name] creates an [agent] from the given parameters. [point] is the point on which the VM is reachabled. [next_available_port] should always provide an available port or raise [Not_found] otherwise. [name] is the name of the agent. [ssh_id] is a path to the private key that will be used for the ssh connection. *) val make : - ?ssh_user:string -> + ?zone:string -> ssh_id:string -> point:string * int -> - binaries_path:string -> + configuration:Configuration.t -> next_available_port:(unit -> int) -> name:string -> unit -> t +(** Encode an agent configuration. *) +val encoding : t Data_encoding.t + (** [name agent] returns the name of the agent. *) val name : t -> string +(** [vm_name agent] returns the vm name of the agent. *) +val vm_name : t -> string + (** [set_name agent name] sets the name of the agent to [name]. *) val set_name : t -> string -> unit -(** [copy agent ~source] copy the file into the [agent] directory and +(** [copy ?destination agent ~source] copy the file into the [agent] directory and returned the directory where the file can be found. It is assumed the [source] file does not exist on the agent machine. If the parent directory does not exist, it will be created. *) -val copy : t -> source:string -> string Lwt.t +val copy : ?destination:string -> t -> source:string -> string Lwt.t (** [next_available_port agent] returns the next available port for this agent. Raises [Not_found] if no port is available. *) @@ -45,5 +51,17 @@ val next_available_port : t -> int (** [runner agent] returns the runner associated with the agent. *) val runner : t -> Runner.t -(** [binaries_path t] the path where binaries should be stored by the agent. *) -val binaries_path : t -> string +(** [point agent] returns the point asociated with the agent. *) +val point : t -> string * int + +(** [configuration t] the configuration of the agent. *) +val configuration : t -> Configuration.t + +(** A wrapper to run a command on the VM of the agent. *) +val cmd_wrapper : t -> Gcloud.cmd_wrapper option + +(** Run a command on the host machine of the VM. *) +val host_run_command : t -> string -> string list -> Process.t + +(** Run a command on the docker image run by the agent. *) +val docker_run_command : t -> string -> string list -> Process.t diff --git a/tezt/lib_cloud/cli.ml b/tezt/lib_cloud/cli.ml index d1f90b14a023735d27e35bd825424a55bc15e54a..508c806d4fb664f9478fe9f6f74777646098d80b 100644 --- a/tezt/lib_cloud/cli.ml +++ b/tezt/lib_cloud/cli.ml @@ -17,6 +17,7 @@ let localhost = Clap.flag ~section ~set_long:"localhost" + ~unset_long:"cloud" ~description:"If set, the test is run locally" false @@ -24,6 +25,7 @@ let monitoring = Clap.flag ~section ~set_long:"monitoring" + ~unset_long:"no-monitoring" ~description: "If set, all the VMs are monitored with netdata. The monitoring \ dashboard is accessible on port 13999 of the monitored VM." @@ -33,6 +35,7 @@ let destroy = Clap.flag ~section ~set_long:"destroy" + ~unset_long:"no-destroy" ~description:"If set, the machines are destroyed at the end of the test" false @@ -61,6 +64,13 @@ let vms = ~description:"Number of VMs running for the test." () +let vm_base_port = + Clap.default_int + ~section + ~long:"vm-base-port" + ~description:"The first available port on the VM" + 30_000 + let ports_per_vm = Clap.default_int ~section @@ -68,52 +78,71 @@ let ports_per_vm = ~description:"Number of opened port per VM (default is 50)" 50 +let proxy = + Clap.flag + ~section + ~set_long:"proxy" + ~description: + "Enables to run the orchestrator on a VM instead of the host machine" + false + let grafana = Clap.flag ~section ~set_long:"grafana" + ~unset_long:"no-grafana" ~description:"Flag to set whether to run grafana" - false + ((not localhost) || proxy) let prometheus = Clap.flag ~section ~set_long:"prometheus" + ~unset_long:"no-prometheus" ~description:"Flag to set whether metrics are exported into prometheus" grafana -let website = +let prometheus_export = Clap.flag ~section - ~set_long:"website" - ~description: - "A webpage is up on localhost to summarize various informations related \ - to the experiment" - prometheus + ~set_long:"prometheus-export" + ~unset_long:"no-prometheus-export" + ~description:"Export a Prometheus snapshot at the end of the scenario" + true -let website_port = +let prometheus_port = Clap.default_int ~section - ~long:"website-port" - ~description:"Set the port used for the website. Default is 8080" - 8080 - -let prometheus_snapshot_directory = - Clap.default_string - ~section - ~long:"prometheus-snapshot-directory" + ~long:"prometheus-port" ~description: - "The directory where the prometheus snapshot can be exported (the \ - default is /tmp)" - "/tmp" + "Set the port on which the prometheus instance will run (default: 9090)." + 9090 -let prometheus_snapshot = +let prometheus_snapshot_filename = Clap.optional_string ~section - ~long:"prometheus-snapshot" + ~long:"prometheus-snapshot-filename" ~description:"Name of the prometheus snapshot file" () +let prometheus_scrape_interval = + Clap.default_int + ~section + ~long:"prometheus-scrape-interval" + ~description: + "Set the scraping interval of the prometheus instance (default: 5)" + 5 + +let website = + Clap.flag + ~section + ~set_long:"website" + ~unset_long:"no-website" + ~description: + "A webpage is up on localhost to summarize various informations related \ + to the experiment" + prometheus + let machine_type = Clap.default_string ~section @@ -123,15 +152,25 @@ let machine_type = https://cloud.google.com/compute/docs/general-purpose-machines#c3d_series)" "n1-standard-2" -let dockerfile = +let dockerfile_alias = Clap.optional_string ~section - ~long:"dockerfile" + ~long:"dockerfile-alias" ~description: - "Specify the name of the dockerfile to use (default is given by the \ - environment variable `TEZT_CLOUD`)" + "Specify the name of the dockerfile alias to use (default is given by \ + the environment variable `TEZT_CLOUD`)" () +let website_port = + Clap.default_int + ~section + ~long:"website-port" + ~description:"Set the port used for the website. Default is 8080" + (* When the website is run by the orchestrator, it is easier to + use port 80. We could decide in the future to use 80 by + default. *) + (if proxy && localhost then 80 else 8080) + let max_run_duration = Clap.default_int ~section @@ -146,4 +185,28 @@ let no_max_run_duration = ~section ~set_long:"no-max-run-duration" ~description:"Ensure the VM can only be destroyed manually." - false + (* If the proxy mode is active, we don't want to use [max_run_duration] + since it aims to run long running tests. *) + proxy + +let tezt_cloud = + Clap.optional_string + ~section + ~long:"tezt-cloud" + ~description:"Overwrite the TEZT_CLOUD variable" + () + +let dns_domain = + Clap.optional_string + ~section + ~long:"dns-domain" + ~description:"DNS domain to use for the website" + () + +let dns = + Clap.flag + ~section + ~set_long:"dns" + ~unset_long:"no-dns" + ~description:"Associate a domain name with the website" + (proxy && website) diff --git a/tezt/lib_cloud/cli.mli b/tezt/lib_cloud/cli.mli index 994c2c86dc541e816c2c7b02035b18c4131d5103..25cf21760eb0b93385b80e427dbec94f97536362 100644 --- a/tezt/lib_cloud/cli.mli +++ b/tezt/lib_cloud/cli.mli @@ -31,6 +31,10 @@ val keep_alive : bool (** [vms] enables to set the number of vms that will be spawned. *) val vms : int option +(** First port opened on the VM. *) +val vm_base_port : int + +(** Number of consecutive ports opened on the VM from the base port. *) val ports_per_vm : int (** When [website] is [true] (default) a website is up for summarizing various @@ -41,16 +45,24 @@ val website : bool [8080]. *) val website_port : int -(** When [prometheus_export] is [true] a Prometheus instance is run locally and +(** When [prometheus] is [true] a Prometheus instance is run locally and metrics of the experiments are exported to prometheus. At the end of the test, the database is snapshotted so that it can be imported later on. *) val prometheus : bool -val grafana : bool +(** If [true], exports a Promtheus snapshot at the end of the scenario. *) +val prometheus_export : bool + +(** Specify the name of the prometheus snapshot. *) +val prometheus_snapshot_filename : string option -val prometheus_snapshot_directory : string +(** Specify the port of the Prometheus instance on the host machine. *) +val prometheus_port : int -val prometheus_snapshot : string option +(** Specify the scraping interval of Prometheus. *) +val prometheus_scrape_interval : int + +val grafana : bool (** Enable to specify a machine type. The string must be a machine description compliant with GCP (ex: "n1-standard-2"). See the different type of machines @@ -61,12 +73,24 @@ val prometheus_snapshot : string option *) val machine_type : string -(** Specify the dockerfile image to use. If not specified, the image +(** Specify the dockerfile alias to use. If not specified, the image name will be given by the value of the variable `TEZT_CLOUD`. *) -val dockerfile : string option +val dockerfile_alias : string option (** Maximum running time of a VM. *) val max_run_duration : int (** Deactivate the max run duration parameter. *) val no_max_run_duration : bool + +(** When [proxy] is [true], the scenario is run via a VM instead of the host + machine. *) +val proxy : bool + +val tezt_cloud : string option + +(** DNS domain to use for the website. *) +val dns_domain : string option + +(** Associate a DNS domain to the website when the proxy mode is used. *) +val dns : bool diff --git a/tezt/lib_cloud/cloud.ml b/tezt/lib_cloud/cloud.ml index ca5d7d5c59471068d3ac435ea4937ebacaa05a6b..3b08bea6b76a1623f4eba8f836dbdc9ff062087c 100644 --- a/tezt/lib_cloud/cloud.ml +++ b/tezt/lib_cloud/cloud.ml @@ -5,8 +5,6 @@ (* *) (*****************************************************************************) -module Configuration = Configuration - (* Tezt-cloud requires to bypass the clean-up process of Tezt. Hence, when the user press Ctrl+C, tezt-cloud needs to catch-up the signal before Tezt. @@ -29,6 +27,15 @@ let sigint = previous_behaviour := previous_handler ; promise +let eof = + let promise, resolver = Lwt.task () in + Lwt.dont_wait + (fun () -> + let* _ = Lwt_io.read_line Lwt_io.stdin in + Lwt.return_unit) + (fun _ -> Lwt.wakeup resolver ()) ; + promise + (* This exception is raised when the test is interrupted by Ctrl+C. *) exception Interrupted @@ -42,26 +49,57 @@ type t = { let shutdown ?exn t = let* () = - if Cli.keep_alive then ( + if Env.keep_alive then ( Log.info "Please press to terminate the scenario." ; let* _ = Lwt_io.read_line Lwt_io.stdin in Lwt.return_unit) else Lwt.return_unit in Log.info "Shutting down processes..." ; - let* () = Process.clean_up () in let* () = - Option.fold - ~none:Lwt.return_unit - ~some:Prometheus.export_snapshot - t.prometheus + Lwt.catch + (fun () -> Process.clean_up ()) + (fun exn -> + Log.warn + "Tezt failed to clean up processes: %s" + (Printexc.to_string exn) ; + Log.warn + "in case you did not destroy VMs, you should execute the 'clean up' \ + job" ; + Lwt.return_unit) + in + let* () = + Lwt.catch + (fun () -> + Option.fold + ~none:Lwt.return_unit + ~some:Prometheus.export_snapshot + t.prometheus) + (fun exn -> + Log.warn "Prometheus snapshot export fails: %s" (Printexc.to_string exn) ; + Lwt.return_unit) in let* () = - Option.fold ~none:Lwt.return_unit ~some:Prometheus.shutdown t.prometheus + Lwt.catch + (fun () -> + Option.fold ~none:Lwt.return_unit ~some:Prometheus.shutdown t.prometheus) + (fun exn -> + Log.warn + "unable to shutdown Prometheus properly: %s" + (Printexc.to_string exn) ; + Lwt.return_unit) in let* () = Option.fold ~none:Lwt.return_unit ~some:Grafana.shutdown t.grafana in + let* () = + if Env.dns then + Gcloud.DNS.remove + ~tezt_cloud:Env.tezt_cloud + ~zone:"tezt-cloud" + ~ip:(Proxy.get_agent t.agents |> Agent.point |> fst) + else Lwt.return_unit + in let* () = Option.fold ~none:Lwt.return_unit @@ -76,35 +114,291 @@ let shutdown ?exn t = | Some exn -> (* The exception is raised to Tezt. *) Lwt.reraise exn (* This function is used to ensure we can connect to the docker image on the VM. *) -let rec wait_ssh_server_running runner = +let wait_ssh_server_running agent = + let runner = Agent.runner agent in + let is_ready _output = true in + let run () = + let cmd, args = + Runner.wrap_with_ssh runner (Runner.Shell.cmd [] "echo" ["-n"; "check"]) + in + Process.spawn cmd (["-o"; "StrictHostKeyChecking=no"] @ args) + in + let* _ = Env.wait_process ~is_ready ~run () in + Lwt.return_unit + +let orchestrator deployement f = + let agents = Deployement.agents deployement in + let* website = + if Env.website then + let* website = Web.start ~agents in + Lwt.return_some website + else Lwt.return_none + in + let* prometheus = + if Env.prometheus then + let* prometheus = Prometheus.start agents in + Lwt.return_some prometheus + else Lwt.return_none + in + let* grafana = + if Env.grafana then + let* grafana = Grafana.run () in + Lwt.return_some grafana + else Lwt.return_none + in + Log.info "Post prometheus" ; + let t = + {website; agents; prometheus; grafana; deployement = Some deployement} + in + let sigint = sigint () in + let main_promise = + (* We also catch error raised from the scenario directly. *) + Lwt.catch + (fun () -> + let* () = f t in + Lwt.return_none) + (fun exn -> Lwt.return_some exn) + in + (* This part is tricky! We want to catch Ctrl+C so that tezt does not + kill all the VMs directly before tezt-cloud termination tasks are + over. When the signal is caught, tezt-cloud takes over. Processes are + cleaned up manually via [Process.clean_up ()]. + *) + let* exn = + Lwt.pick + [ + (let* () = sigint in + Lwt.return_some Interrupted); + main_promise; + ] + in + shutdown ?exn t + +let attach agent = + let runner = Agent.runner agent in + let hooks = + Process. + { + on_log = + (fun str -> Log.debug ~color:Log.Color.reset "%s" (String.trim str)); + on_spawn = (fun _ _ -> ()); + } + in + let has_sigint = ref false in + let on_sigint = + let* () = sigint () in + has_sigint := true ; + Log.debug "Sigint triggered..." ; + let cmd, args = + Runner.wrap_with_ssh + runner + (Runner.Shell.cmd [] "screen" ["-S"; "tezt-cloud"; "-X"; "stuff"; "^C"]) + in + let* () = + Process.spawn ~hooks cmd (["-o"; "StrictHostKeyChecking=no"] @ args) + |> Process.check + in + let cmd, args = + Runner.wrap_with_ssh + runner + (Runner.Shell.cmd [] "stdbuf" ["-oL"; "tail"; "-f"; "screenlog.0"]) + in + let _p = + Process.spawn ~hooks cmd (["-o"; "StrictHostKeyChecking=no"] @ args) + |> Process.check + in + let* _ = eof in + let* () = + let process = Process.spawn ~runner "pkill" ["screen"] in + let* _ = Process.wait process in + Lwt.return_unit + in + let* () = + let process = Process.spawn ~runner "rm" ["-f"; "screenlog.0"] in + let* _ = Process.wait process in + Lwt.return_unit + in + let* () = + let process = + Process.spawn "rm" [Path.proxy_deployement ~tezt_cloud:Env.tezt_cloud] + in + let* _ = Process.wait process in + Lwt.return_unit + in + Lwt.return_unit + in + let on_eof = + let* () = eof in + Log.info "Detach from the proxy process." ; + if !has_sigint then on_sigint + else + let* uri = + if Env.dns then + let* domain = + Gcloud.DNS.get_domain ~tezt_cloud:Env.tezt_cloud ~zone:"tezt-cloud" + in + Lwt.return (Format.asprintf "http://%s" domain) + else Lwt.return (Format.asprintf "http://%s" (Agent.point agent |> fst)) + in + Log.info "Deployement website can be accessed here: %s" uri ; + Lwt.return_unit + in + Log.Style.set_prefix Log.Style.Hidden ; + Log.Style.set_timestamp Log.Style.Hidden ; let cmd, args = - Runner.wrap_with_ssh runner (Runner.Shell.cmd [] "echo" ["-n"; "check"]) + Runner.wrap_with_ssh + runner + (Runner.Shell.cmd [] "stdbuf" ["-oL"; "tail"; "-f"; "screenlog.0"]) + in + let logger = + Lwt.catch + (fun () -> + let* () = + Process.spawn ~hooks cmd (["-o"; "StrictHostKeyChecking=no"] @ args) + |> Process.check + in + Lwt.return_unit) + (fun exn -> + Log.debug "Interrupted: %s" (Printexc.to_string exn) ; + on_sigint) + in + Lwt.choose [logger; on_sigint; on_eof] + +let try_reattach () = + let tezt_cloud = Env.tezt_cloud in + if Sys.file_exists (Path.proxy_deployement ~tezt_cloud) then + let contents = Base.read_file (Path.proxy_deployement ~tezt_cloud) in + let json = Data_encoding.Json.from_string contents |> Result.get_ok in + let deployement = + Data_encoding.Json.destruct (Data_encoding.list Agent.encoding) json + |> Deployement.of_agents + in + let agents = Deployement.agents deployement in + let proxy_agent = + agents + |> List.find (fun agent -> + let proxy_agent_prefix = + Format.asprintf "%s-proxy" Env.tezt_cloud + in + String.starts_with ~prefix:proxy_agent_prefix (Agent.name agent)) + in + let* is_ssh_server_running = + Lwt.pick + [ + (let* () = Lwt_unix.sleep 5. in + Lwt.return_false); + (let* () = wait_ssh_server_running proxy_agent in + Lwt.return_true); + ] + in + if is_ssh_server_running then + let* is_running = + Lwt.catch + (fun () -> + let* status = + Process.spawn + ~runner:(Agent.runner proxy_agent) + "ls" + ["screenlog.0"] + |> Process.wait + in + match status with + | WEXITED 0 -> Lwt.return_true + | _ -> Lwt.return_false) + (fun _exn -> Lwt.return_false) + in + if is_running then + let* () = + Lwt.catch + (fun () -> attach proxy_agent) + (fun exn -> + Log.debug "INFO: %s" (Printexc.to_string exn) ; + Lwt.return_unit) + in + Lwt.return_true + else Lwt.return_false + else Lwt.return_false + else Lwt.return_false + +let init_proxy ?(proxy_files = []) deployement = + let agents = Deployement.agents deployement in + let proxy_agent = + agents + |> List.find (fun agent -> + let proxy_agent_prefix = Format.asprintf "%s-proxy" Env.tezt_cloud in + String.starts_with ~prefix:proxy_agent_prefix (Agent.name agent)) + in + let* () = wait_ssh_server_running proxy_agent in + let destination = + (Agent.configuration proxy_agent).binaries_path + // Filename.basename Path.self + in + let* self = Agent.copy ~destination proxy_agent ~source:Path.self in + let json = + Data_encoding.Json.construct (Data_encoding.list Agent.encoding) agents + in + let contents = Data_encoding.Json.to_string json in + let tezt_cloud = Env.tezt_cloud in + let proxy_deployement = Path.proxy_deployement ~tezt_cloud in + Base.write_file proxy_deployement ~contents ; + let* () = + Proxy.copy_files proxy_agent ~scenario_files:proxy_files ~proxy_deployement + in + let runner = Agent.runner proxy_agent in + let* () = + (* This should not be necessary, this is to ensure there is no [screen] leftover. *) + let process = Process.spawn ~runner "pkill" ["screen"] in + let* _ = Process.wait process in + Lwt.return_unit + in + let* () = + (* We start a screen session in detached mode. The orchestrator will run in this session. *) + Process.spawn ~runner "screen" ["-S"; "tezt-cloud"; "-d"; "-m"] + |> Process.check + in + let process = + let args = + let args = Sys.argv |> Array.to_list |> List.tl in + args @ ["--localhost"; "--tezt-cloud"; Env.tezt_cloud] + (* [--localhost] will be combined with --proxy, this enables to detect we want to run in [`Orchestrator]. + + [--tezt-cloud] is used so that the [`Orchestrator] mode knows this value. + *) + in + (* We execute a command in a screen session that will start the orchestrator. *) + Agent.docker_run_command + proxy_agent + "screen" + (["-S"; "tezt-cloud"; "-X"; "exec"] @ (self :: args)) + in + let* () = + Agent.docker_run_command + proxy_agent + "screen" + ["-S"; "tezt-cloud"; "-X"; "log"] + |> Process.check + in + let* () = + Lwt.catch + (fun () -> attach proxy_agent) + (fun exn -> + Test.fail "Unable to attach process: %s" (Printexc.to_string exn)) in - let* status = - Process.spawn cmd (["-o"; "StrictHostKeyChecking=no"] @ args) - |> Process.wait - in - match status with - | Unix.WEXITED 0 -> Lwt.return_unit - | _ -> - Log.info - "SSH server is not running, let's wait 2 seconds and check again..." ; - let* () = Lwt_unix.sleep 2. in - wait_ssh_server_running runner - -let register ?(docker_push = not Cli.localhost) ?vms ~__FILE__ ~title ~tags - ?seed f = - Test.register ~__FILE__ ~title ~tags ?seed @@ fun () -> let* () = - if docker_push then - let* () = Jobs.deploy_docker_registry () in - Jobs.docker_build ~push:true () - else Jobs.docker_build ~push:false () + let* status = Process.wait process in + match status with + | WEXITED 0 -> Lwt.return_unit + | _ -> Test.fail "Proxy scenario has failed" in + if Env.destroy then Deployement.terminate deployement else Lwt.return_unit + +let register ?proxy_files ?vms ~__FILE__ ~title ~tags ?seed f = + Test.register ~__FILE__ ~title ~tags ?seed @@ fun () -> + let* () = Env.init () in let vms = (* The Cli arguments by-pass the argument given here. This enable the user to always have decide precisely the number of vms to be run. *) - match (vms, Cli.vms) with + match (vms, Env.vms) with | None, None -> None | None, Some i | Some _, Some i -> let vms = List.init i (fun _ -> Configuration.make ()) in @@ -122,86 +416,69 @@ let register ?(docker_push = not Cli.localhost) ?vms ~__FILE__ ~title ~tags prometheus = None; deployement = None; } - | Some configurations -> - let ports_per_vm = Cli.ports_per_vm in - let* deployement = - Deployement.deploy - ~ports_per_vm - ~configurations - ~localhost:Cli.localhost - () - in - let agents = Deployement.agents deployement in - let* () = - agents - |> List.map (fun agent -> Agent.runner agent |> wait_ssh_server_running) - |> Lwt.join - in - if Cli.monitoring then ( - Log.info "Monitoring enabled" ; - agents - |> List.iter (fun agent -> - let address = - Agent.runner agent |> fun runner -> - Runner.address (Some runner) - in - Log.info - "Monitoring of agent %s can be accessed at 'http://%s:19999'" - (Agent.name agent) - address)) ; - let* website = - if Cli.website then - let* website = - Web.start ~port:Cli.website_port ~deployement ~agents + | Some configurations -> ( + let tezt_cloud = Env.tezt_cloud in + match Env.mode with + | `Orchestrator -> + (* The scenario is executed locally on the proxy VM. *) + let contents = Base.read_file (Path.proxy_deployement ~tezt_cloud) in + let json = Data_encoding.Json.from_string contents |> Result.get_ok in + let deployement = + Data_encoding.Json.destruct (Data_encoding.list Agent.encoding) json + |> Deployement.of_agents in - Lwt.return_some website - else Lwt.return_none - in - let* prometheus = - if Cli.prometheus then - let* prometheus = Prometheus.start agents in - Lwt.return_some prometheus - else Lwt.return_none - in - let* grafana = - if Cli.grafana then - let* grafana = Grafana.run () in - Lwt.return_some grafana - else Lwt.return_none - in - let t = - {website; agents; prometheus; grafana; deployement = Some deployement} - in - let sigint = sigint () in - let main_promise = - (* We also catch error raised from the scenario directly. *) - Lwt.catch - (fun () -> - let* () = f t in - Lwt.return_none) - (fun exn -> Lwt.return_some exn) - in - (* This part is tricky! We want to catch Ctrl+C so that tezt does not - kill all the VMs directly before tezt-cloud termination tasks are - over. When the signal is caught, tezt-cloud takes over. Processes are - cleaned up manually via [Process.clean_up ()]. - *) - let* exn = - Lwt.pick - [ - (let* () = sigint in - Lwt.return_some Interrupted); - main_promise; - ] - in - shutdown ?exn t + let* () = + Deployement.agents deployement + |> List.map wait_ssh_server_running + |> Lwt.join + in + orchestrator deployement f + | `Localhost -> + (* The scenario is executed locally and the VM are on the host machine. *) + let* () = Jobs.docker_build ~push:false () in + let* deployement = Deployement.deploy ~configurations in + let* () = + Deployement.agents deployement + |> List.map wait_ssh_server_running + |> Lwt.join + in + orchestrator deployement f + | `Cloud -> + (* The scenario is executed locally and the VMs are on the cloud. *) + let* () = Jobs.deploy_docker_registry () in + let* () = Jobs.docker_build ~push:true () in + let* deployement = Deployement.deploy ~configurations in + let* () = + Deployement.agents deployement + |> List.map wait_ssh_server_running + |> Lwt.join + in + orchestrator deployement f + | `Host -> + (* The scenario is executed remotely. *) + let* proxy_running = try_reattach () in + if not proxy_running then + let* () = Jobs.deploy_docker_registry () in + let* () = Jobs.docker_build ~push:true () in + let* deployement = Deployement.deploy ~configurations in + let* () = + Deployement.agents deployement + |> List.map wait_ssh_server_running + |> Lwt.join + in + init_proxy ?proxy_files deployement + else Lwt.return_unit) -let agents t = t.agents +let agents t = + match Env.mode with + | `Orchestrator -> + let proxy_agent = Proxy.get_agent t.agents in + let proxy_vm_name = Agent.vm_name proxy_agent in + t.agents + |> List.filter (fun agent -> Agent.vm_name agent <> proxy_vm_name) + | `Host | `Cloud | `Localhost -> t.agents -let get_configuration t agent = - match t.deployement with - | None -> Configuration.make () - | Some deployement -> Deployement.get_configuration deployement agent +let get_configuration = Agent.configuration let set_agent_name t agent name = Agent.set_name agent name ; diff --git a/tezt/lib_cloud/cloud.mli b/tezt/lib_cloud/cloud.mli index 6c00367087aebeeb8e6906a07c5cd5de92481a94..01b3425513c3e8eac09bcb09c17173d024824cba 100644 --- a/tezt/lib_cloud/cloud.mli +++ b/tezt/lib_cloud/cloud.mli @@ -5,17 +5,13 @@ (* *) (*****************************************************************************) -module Configuration : sig - include module type of Configuration -end - type t (** [register ?docker_push ?vms] is a wrapper around [Test.register]. It enables to run a test that can use machines deployed onto the cloud. *) val register : - ?docker_push:bool -> + ?proxy_files:string list -> ?vms:Configuration.t list -> __FILE__:string -> title:string -> @@ -26,7 +22,7 @@ val register : val agents : t -> Agent.t list -val get_configuration : t -> Agent.t -> Configuration.t +val get_configuration : Agent.t -> Configuration.t val push_metric : t -> ?labels:(string * string) list -> name:string -> float -> unit diff --git a/tezt/lib_cloud/configuration.ml b/tezt/lib_cloud/configuration.ml index e2a27d55961dc2214d9de0ac6a7380fb6fe9a56d..424dc0aa9ca290a133456b599772a4f5ad5bb84b 100644 --- a/tezt/lib_cloud/configuration.ml +++ b/tezt/lib_cloud/configuration.ml @@ -5,41 +5,30 @@ (* *) (*****************************************************************************) -type docker_image = - | Custom of {tezt_cloud : string} - | Image of {docker_image : string} - -let string_of_docker_image ~project_id = function - | Custom {tezt_cloud} -> - Env.custom_docker_image ~docker_image_name:tezt_cloud ~project_id () - | Image {docker_image} -> docker_image - type t = { machine_type : string; - docker_image : docker_image; + docker_image : Env.docker_image; max_run_duration : int option; + binaries_path : string; } -let default_docker_image = - match (Cli.dockerfile, Cli.localhost) with - | None, false -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - Custom {tezt_cloud} - | None, true -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - let docker_image = Format.asprintf "%s:latest" tezt_cloud in - Image {docker_image} - | Some tezt_cloud, false -> Custom {tezt_cloud} - | Some tezt_cloud, true -> - let docker_image = Format.asprintf "%s:latest" tezt_cloud in - Image {docker_image} - -let make ?max_run_duration ?(machine_type = Cli.machine_type) ?docker_image () = - let docker_image = Option.value ~default:default_docker_image docker_image in +let make ?binaries_path ?max_run_duration ?machine_type ?docker_image () = + (* We don't use [env] because of dependency issue: We want the user of the + library to use this value function the [env] value is created. *) + let docker_image = Option.value ~default:Env.docker_image docker_image in + let machine_type = Option.value ~default:Env.machine_type machine_type in + let default_binaries_path = + match docker_image with + | Env.Gcp _ -> Path.default_binaries_path () + | Octez_latest_release -> "/usr/local/bin" + in + let binaries_path = + Option.value ~default:default_binaries_path binaries_path + in let max_run_duration = match max_run_duration with | None -> - if Cli.no_max_run_duration then None else Some Cli.max_run_duration + if Env.no_max_run_duration then None else Some Env.max_run_duration | Some max_run_duration -> Some max_run_duration in - {machine_type; docker_image; max_run_duration} + {machine_type; docker_image; max_run_duration; binaries_path} diff --git a/tezt/lib_cloud/configuration.mli b/tezt/lib_cloud/configuration.mli index d55a3a96b0872051e4e3edfea3bcd5e8f949f73c..bf0f894528f3777484669a3d5f759129a5f1d019 100644 --- a/tezt/lib_cloud/configuration.mli +++ b/tezt/lib_cloud/configuration.mli @@ -5,21 +5,17 @@ (* *) (*****************************************************************************) -type docker_image = - | Custom of {tezt_cloud : string} - | Image of {docker_image : string} - -val string_of_docker_image : project_id:string -> docker_image -> string - type t = private { machine_type : string; - docker_image : docker_image; + docker_image : Env.docker_image; max_run_duration : int option; + binaries_path : string; } val make : + ?binaries_path:string -> ?max_run_duration:int -> ?machine_type:string -> - ?docker_image:docker_image -> + ?docker_image:Env.docker_image -> unit -> t diff --git a/tezt/lib_cloud/deployement.ml b/tezt/lib_cloud/deployement.ml index 70b33c428b29965e55aa2eaa5d1a7454800b6d88..d1e5e4ed10291b0e4a99f51fa357b51080ffc905 100644 --- a/tezt/lib_cloud/deployement.ml +++ b/tezt/lib_cloud/deployement.ml @@ -6,77 +6,54 @@ (*****************************************************************************) module Remote = struct - type workspace_info = {configuration : Configuration.t; number_of_vms : int} - type point_info = {workspace_name : string; gcp_name : string} - type address = string - - type t = { - base_port : int; - ports_per_vm : int; - agents_info : (address, point_info) Hashtbl.t; - agents : Agent.t list; - workspaces_info : (string, workspace_info) Hashtbl.t; - zone : string; - } + type t = {agents : Agent.t list} - let rec wait_docker_running ~zone ~vm_name = - let*? process = + let wait_docker_running ~vm_name () = + let ssh_private_key_filename = Env.ssh_private_key_filename () in + let* zone = Env.zone () in + let is_ready _output = true in + let run () = + (* Try to get the docker images up. *) Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["ps"; "--format"; "{{.Names}}"] in - let* status = Process.wait process in - match status with - | Unix.WEXITED 0 -> - let* images_name = Process.check_and_read_stdout process in - String.split_on_char '\n' images_name - |> List.filter (fun s -> s <> "") - |> List.map (fun image_name -> - let*? process = - Gcloud.compute_ssh - ~zone - ~vm_name - "docker" - ["inspect"; "--format"; "{{.State.Running}}"; image_name] - in - let* status = Process.wait process in - match status with - | Unix.WEXITED 0 -> - let* is_running = - let* status = Process.check_and_read_stdout process in - String.trim status |> Stdlib.bool_of_string |> Lwt.return - in - if is_running then Lwt.return_unit - else ( - Log.info - "Docker image is not ready, let's wait 2 seconds and \ - check again..." ; - wait_docker_running ~zone ~vm_name) - | _ -> - Log.info - "Docker image is not ready, let's wait 2 seconds and \ - check again..." ; - let* () = Lwt_unix.sleep 2. in - wait_docker_running ~zone ~vm_name) - |> Lwt.join - | _ -> - Log.info - "Docker image is not ready, let's wait 2 seconds and check again..." ; - let* () = Lwt_unix.sleep 2. in - wait_docker_running ~zone ~vm_name + let* output = Env.wait_process ~is_ready ~run () in + let images_name = + output |> String.split_on_char '\n' |> List.filter (fun s -> s <> "") + in + let is_ready output = String.trim output |> Stdlib.bool_of_string in + let run image_name () = + (* Try to get the docker images that are actually running. *) + Gcloud.compute_ssh + ~zone + ~vm_name + ~ssh_private_key_filename + "docker" + ["inspect"; "--format"; "{{.State.Running}}"; image_name] + in + let* _ = + images_name + |> List.map (fun image_name -> + Env.wait_process ~is_ready ~run:(run image_name) ()) + |> Lwt.all + in + unit - let workspace_deploy ?(base_port = 30_000) ?(ports_per_vm = 50) - ~max_run_duration ~workspace_name ~machine_type ~number_of_vms - ~docker_image () = + let workspace_deploy ~workspace_name ~number_of_vms ~configuration = let* () = Terraform.VM.Workspace.select workspace_name in - let* project_id = Gcloud.project_id () in - let docker_image = - Configuration.string_of_docker_image ~project_id docker_image + let* docker_image = + Env.uri_of_docker_image configuration.Configuration.docker_image in + let machine_type = configuration.machine_type in + let max_run_duration = configuration.max_run_duration in + let ports_per_vm = Env.ports_per_vm in + let base_port = Env.vm_base_port in let* () = Terraform.VM.deploy ~max_run_duration @@ -90,26 +67,14 @@ module Remote = struct List.init number_of_vms (fun i -> Format.asprintf "%s-%03d" workspace_name (i + 1)) in - let* zone = Terraform.VM.zone () in + let* zone = Env.zone () in let* () = - List.map (fun vm_name -> wait_docker_running ~zone ~vm_name) names + List.map (fun vm_name -> wait_docker_running ~vm_name ()) names |> Lwt.join in - let* () = - let run_command ~vm_name cmd args = - Gcloud.compute_ssh ~zone ~vm_name cmd args - in - if Cli.monitoring then - List.map - (fun vm_name -> Monitoring.run ~run_command:(run_command ~vm_name)) - names - |> Lwt.join - else Lwt.return_unit - in - let ssh_id = Lazy.force Env.ssh_private_key in - let binaries_path = Path.default_binaries_path () in - let agent_of_name name = - let* ip = Gcloud.get_ip_address_from_name ~zone name in + let ssh_private_key_filename = Env.ssh_private_key_filename () in + let make_agent vm_name = + let* ip = Gcloud.get_ip_address_from_name ~zone vm_name in let point = (ip, base_port) in let next_available_port = let port = ref base_port in @@ -117,25 +82,31 @@ module Remote = struct incr port ; !port in - Agent.make ~ssh_id ~point ~binaries_path ~next_available_port ~name () + let cmd_wrapper = + Gcloud.cmd_wrapper ~zone ~ssh_private_key_filename ~vm_name + in + let* () = + if Env.monitoring then Monitoring.run ~cmd_wrapper () + else Lwt.return_unit + in + Agent.make + ~ssh_id:ssh_private_key_filename + ~zone + ~point + ~configuration + ~next_available_port + ~name:vm_name + () |> Lwt.return in - let* agents = names |> Lwt_list.map_p agent_of_name in + let* agents = names |> Lwt_list.map_p make_agent in Lwt.return agents - let get_configuration agents_info workspaces_info agent = - let address = agent |> Agent.runner |> Option.some |> Runner.address in - let {workspace_name; _} = Hashtbl.find agents_info address in - let {configuration; _} = Hashtbl.find workspaces_info workspace_name in - configuration - - let order_agents agents_info workspaces_info agents configurations = + let order_agents agents configurations = let bindings = agents |> List.map (fun agent -> - let configuration = - get_configuration agents_info workspaces_info agent - in + let configuration = Agent.configuration agent in (configuration, agent)) in let rec order configurations bindings = @@ -148,48 +119,48 @@ module Remote = struct in order configurations bindings - let deploy ?(base_port = 30_000) ?(ports_per_vm = 50) ~configurations () = - let tezt_cloud = Lazy.force Env.tezt_cloud in - let workspaces_info = Hashtbl.create 11 in + let deploy_proxy () = + let workspace_name = Format.asprintf "%s-proxy" Env.tezt_cloud in + let configuration = Configuration.make () in + let tezt_cloud = Env.tezt_cloud in + let* () = Terraform.VM.Workspace.init ~tezt_cloud [workspace_name] in + let* agents = + workspace_deploy ~workspace_name ~configuration ~number_of_vms:1 + in + match agents with [agent] -> Lwt.return agent | _ -> assert false + + (* + Deployement requires to create new VMs and organizing them per group of + configuration. Each configuration leads to one terraform workspace. + *) + let deploy ~proxy ~configurations = let agents_info = Hashtbl.create 11 in - let () = + let workspaces_info = + (* VMs are grouped per group of configuration. Each group leads to one workspace. *) List.to_seq configurations |> Seq.group ( = ) - |> Seq.iteri (fun i seq -> + |> Seq.mapi (fun i seq -> let configuration = Seq.uncons seq |> Option.get |> fst in - let name = Format.asprintf "%s-%d" tezt_cloud i in - Hashtbl.add - workspaces_info - name - {configuration; number_of_vms = Seq.length seq}) + let workspace_name = Format.asprintf "%s-%d" Env.tezt_cloud i in + (workspace_name, (configuration, Seq.length seq))) in let* () = Terraform.Docker_registry.init () in let* () = Terraform.VM.init () in - let workspaces_names = - workspaces_info |> Hashtbl.to_seq_keys |> List.of_seq - in - let* () = Terraform.VM.Workspace.init workspaces_names in + let workspaces_names = workspaces_info |> Seq.map fst |> List.of_seq in + let tezt_cloud = Env.tezt_cloud in + let* () = Terraform.VM.Workspace.init ~tezt_cloud workspaces_names in let* agents = - workspaces_info |> Hashtbl.to_seq |> List.of_seq - |> Lwt_list.map_s - (fun - ( workspace_name, - { - configuration = {machine_type; docker_image; max_run_duration}; - number_of_vms; - } ) - -> + workspaces_info |> List.of_seq + |> Lwt_list.map_s (fun (workspace_name, (configuration, number_of_vms)) -> + let* () = + Jobs.docker_build + ~docker_image:configuration.Configuration.docker_image + ~push:true + () + in let* () = Terraform.VM.Workspace.select workspace_name in let* () = Terraform.VM.init () in let* agents = - workspace_deploy - ~base_port - ~ports_per_vm - ~workspace_name - ~max_run_duration - ~machine_type - ~number_of_vms - ~docker_image - () + workspace_deploy ~workspace_name ~number_of_vms ~configuration in agents |> List.iter (fun agent -> @@ -204,27 +175,29 @@ module Remote = struct Lwt.return agents) in let agents = - order_agents - agents_info - workspaces_info - (List.concat agents) - configurations + (* We want to ensure agents are given in the same order than the + configurations. *) + order_agents (List.concat agents) configurations in - let* zone = Terraform.VM.zone () in - Lwt.return - {base_port; ports_per_vm; zone; agents_info; agents; workspaces_info} - - let get_configuration t agent = - get_configuration t.agents_info t.workspaces_info agent - - let run_vm_command {agents_info; zone; _} agent cmd args = - let address = agent |> Agent.runner |> Option.some |> Runner.address in - let {gcp_name = vm_name; _} = Hashtbl.find agents_info address in - Gcloud.compute_ssh ~zone ~vm_name cmd args + let* t = + if proxy then + let* agent = deploy_proxy () in + let* () = + if Env.dns then + Gcloud.DNS.add + ~tezt_cloud:Env.tezt_cloud + ~zone:"tezt-cloud" + ~ip:(Agent.point agent |> fst) + else Lwt.return_unit + in + Lwt.return {agents = agent :: agents} + else Lwt.return {agents} + in + Lwt.return t let agents t = t.agents - let terminate ?exn t = + let terminate ?exn _t = (match exn with | None -> Log.report ~color:Log.Color.FG.green "Scenario ended successfully." @@ -233,11 +206,13 @@ module Remote = struct ~color:Log.Color.FG.red "Scenario terminated unexpectedly:'%s'" (Printexc.to_string exn)) ; - if Cli.destroy then ( + if Env.destroy then ( Log.report "Destroying VMs, this may take a while..." ; - let workspaces = Hashtbl.to_seq_keys t.workspaces_info |> List.of_seq in - let* () = Terraform.VM.destroy workspaces in - Terraform.VM.Workspace.destroy ()) + let tezt_cloud = Env.tezt_cloud in + let* workspaces = Terraform.VM.Workspace.list ~tezt_cloud in + let* project_id = Gcloud.project_id () in + let* () = Terraform.VM.destroy workspaces ~project_id in + Terraform.VM.Workspace.destroy ~tezt_cloud) else ( Log.report "No VM destroyed! Don't forget to destroy them when you are done with \ @@ -251,35 +226,34 @@ module Localhost = struct processes : Process.t list; base_port : int; ports_per_vm : int; - names : (string, string) Hashtbl.t; agents : Agent.t list; } - let deploy ?(base_port = 30_000) ?(ports_per_vm = 50) ~configurations () = - (* We need to intialize the docker registry even on localhost to fetch the - docker image. *) - let* () = Terraform.Docker_registry.init () in - let* project_id = Gcloud.project_id () in + let vm_name i = Format.asprintf "%s-%03d" Env.tezt_cloud (i + 1) + + let deploy ~configurations () = let number_of_vms = List.length configurations in - let tezt_cloud = Lazy.force Env.tezt_cloud in - let names = Hashtbl.create 11 in - (* The current configuration is actually unused in localhost. Only the - number of VMs matters. *) - let processes = + let base_port = Env.vm_base_port in + let ports_per_vm = Env.ports_per_vm in + let* processes = List.to_seq configurations |> Seq.mapi (fun i configuration -> - let name = Format.asprintf "%s-%03d" tezt_cloud (i + 1) in + let name = vm_name i in let start = base_port + (i * ports_per_vm) |> string_of_int in let stop = base_port + ((i + 1) * ports_per_vm) - 1 |> string_of_int in let publish_ports = (start, stop, start, stop) in - let docker_image = - Configuration.string_of_docker_image - ~project_id - configuration.Configuration.docker_image + let* () = + Jobs.docker_build + ~docker_image:configuration.Configuration.docker_image + ~push:false + () in - let*? process = + let* docker_image = + Env.uri_of_docker_image configuration.docker_image + in + let process = Docker.run ~rm:true ~name @@ -288,8 +262,8 @@ module Localhost = struct docker_image ["-D"; "-p"; start; "-e"] in - process) - |> List.of_seq + Lwt.return process) + |> List.of_seq |> Lwt.all in (* We need to wait a little the machine is up. As for the remote case, we could be more robust to handle that. *) @@ -299,15 +273,7 @@ module Localhost = struct |> Seq.iter (fun i -> let port = base_port + (i * ports_per_vm) in Hashtbl.replace next_port ("localhost", port) (port + 1)) ; - let* () = - let run_command cmd args = - let value : Process.t = Process.spawn cmd args in - let run process = Process.check_and_read_stdout process in - {value; run} - in - if Cli.monitoring then Monitoring.run ~run_command else Lwt.return_unit - in - let ssh_id = Lazy.force Env.ssh_private_key in + let ssh_id = Env.ssh_private_key_filename () in let get_point i = let port = base_port + (i * ports_per_vm) in ("localhost", port) @@ -317,36 +283,25 @@ module Localhost = struct Hashtbl.replace next_port point (port + 1) ; port in - let binaries_path = Path.default_binaries_path () in + let* () = if Env.monitoring then Monitoring.run () else Lwt.return_unit in let agents = - List.init number_of_vms (fun i -> - let name = Format.asprintf "localhost_docker_%d" i in - let point = get_point i in - Agent.make - ~ssh_id - ~point - ~binaries_path - ~next_available_port:(fun () -> next_port point) - ~name - ()) + configurations + |> List.mapi (fun i configuration -> + let name = Format.asprintf "localhost_docker_%d" i in + let point = get_point i in + Agent.make + ~ssh_id + ~point + ~configuration + ~next_available_port:(fun () -> next_port point) + ~name + ()) in - Lwt.return - {number_of_vms; processes; base_port; ports_per_vm; names; agents} - - (* Since in [localhost] mode, the VM is the host machine, this comes back - to just run a command on the host machine. *) - let run_vm_command cmd args = - let value : Process.t = Process.spawn cmd args in - let run process = Process.check_and_read_stdout process in - {value; run} + Lwt.return {number_of_vms; processes; base_port; ports_per_vm; agents} let agents t = t.agents - let get_configuration _t _agent = - (* The configuration is not used in localhost. *) - Configuration.make () - - let terminate ?exn _t = + let terminate ?exn t = (match exn with | None -> Log.report ~color:Log.Color.FG.green "Scenario ended successfully." @@ -357,46 +312,41 @@ module Localhost = struct (Printexc.to_string exn)) ; Log.report "Terminate test: tearing down docker containers..." ; let* () = - (* The netdata docker is run in the background. It must be shut - explicitely. *) - if Cli.monitoring then - let*! () = Docker.kill "netdata" in - let*! () = Docker.rm "netdata" in - Lwt.return_unit - else Lwt.return_unit + t.agents + |> List.mapi (fun i _agent -> + let* _ = Docker.kill (vm_name i) |> Process.wait in + Lwt.return_unit) + |> Lwt.join in - Lwt.return_unit + if Env.monitoring then + let* () = Docker.kill "netdata" |> Process.check in + Lwt.return_unit + else Lwt.return_unit end type t = Remote of Remote.t | Localhost of Localhost.t -let deploy ?(base_port = 30_000) ?(ports_per_vm = 50) ~configurations ~localhost - () = - if localhost then - let* localhost = - Localhost.deploy ~base_port ~ports_per_vm ~configurations () - in - Lwt.return (Localhost localhost) - else - let* remote = Remote.deploy ~base_port ~ports_per_vm ~configurations () in - Lwt.return (Remote remote) - -let run_vm_command t agent cmd args = - match t with - | Remote remote -> Remote.run_vm_command remote agent cmd args - | Localhost _localhost -> Localhost.run_vm_command cmd args +let deploy ~configurations = + match Env.mode with + | `Localhost -> + let* localhost = Localhost.deploy ~configurations () in + Lwt.return (Localhost localhost) + | `Cloud -> + let* remote = Remote.deploy ~proxy:false ~configurations in + Lwt.return (Remote remote) + | `Host -> + let* remote = Remote.deploy ~proxy:true ~configurations in + Lwt.return (Remote remote) + | `Orchestrator -> assert false let agents t = match t with | Remote remote -> Remote.agents remote | Localhost localhost -> Localhost.agents localhost -let get_configuration t = - match t with - | Remote remote -> Remote.get_configuration remote - | Localhost localhost -> Localhost.get_configuration localhost - let terminate ?exn t = match t with | Remote remote -> Remote.terminate ?exn remote | Localhost localhost -> Localhost.terminate ?exn localhost + +let of_agents agents = Remote {agents} diff --git a/tezt/lib_cloud/deployement.mli b/tezt/lib_cloud/deployement.mli index 08f2e301d1fc09b1ddb7a42630a562b90aafbfca..4de1bb13ffffbdf04d906220750391b3e4dab032 100644 --- a/tezt/lib_cloud/deployement.mli +++ b/tezt/lib_cloud/deployement.mli @@ -8,32 +8,20 @@ (** Type for resources managed by a deployment. *) type t -(** [deploy ?base_port ?ports_per_vm ~max_run_duration ~configuration ~localhost ()] deploys the +(** [deploy ~configurations] deploys the expected number of vms. For each vm, we can specify a [base_port] which is the first port to be opened and [ports_per_vm] specify the number of opened port from the [base_port]. The promise returned by this function is fulfilled when all the vms are deployed. Consequently, it can take some times. *) -val deploy : - ?base_port:int -> - ?ports_per_vm:int -> - configurations:Configuration.t list -> - localhost:bool -> - unit -> - t Lwt.t +val deploy : configurations:Configuration.t list -> t Lwt.t (** [get_agents t] returns the list of agents deployed. *) val agents : t -> Agent.t list -val get_configuration : t -> Agent.t -> Configuration.t - -(** [run_vm_command t ~address cmd args] can run a command on the vm located at - [address]. This is different from running a commend on the agent directly - since the agent runs on a docker image. *) -val run_vm_command : - t -> Agent.t -> string -> string list -> (Process.t, string) Runnable.t - -(** [terminate ?exn t] should be called to tear down the machine. Do note +(** [terminate ?exn env t] should be called to tear down the machine. Do note that this call may or may not destroy the machine depending on what the user has chosen (see [Cli.destroy].*) val terminate : ?exn:exn -> t -> unit Lwt.t + +val of_agents : Agent.t list -> t diff --git a/tezt/lib_cloud/docker.ml b/tezt/lib_cloud/docker.ml index c904dec0cf27572ed7b3c583d032a917fc8c64d3..a2bf45398f0e8e3bcc08f3ad8a2e2db4589dc522 100644 --- a/tezt/lib_cloud/docker.ml +++ b/tezt/lib_cloud/docker.ml @@ -4,56 +4,48 @@ (* SPDX-FileCopyrightText: 2024 Nomadic Labs *) (* *) (*****************************************************************************) + let name = "docker" let color = Log.Color.FG.yellow -let build ?(tag = "latest") ?dockerfile ~args () = +let build ?image_name ?alias ?(tag = "latest") ?dockerfile ~args () = let build_args = args |> List.map (fun (key, value) -> ["--build-arg"; Format.asprintf "%s=%s" key value]) |> List.concat in - let tezt_cloud = Lazy.force Env.tezt_cloud in - let dockerfile_path = Lazy.force Env.dockerfile in - let dockerfile = Option.value ~default:dockerfile_path dockerfile in - let tag = ["-t"; Format.asprintf "%s:%s" tezt_cloud tag] in + let alias = Option.value ~default:Env.dockerfile_alias alias in + let dockerfile = Option.value ~default:(Path.dockerfile ~alias) dockerfile in + let image_name = Option.value ~default:alias image_name in + let tag = ["-t"; Format.asprintf "%s:%s" image_name tag] in let args = ["build"; "-f"; dockerfile] @ build_args @ tag @ ["."] in - let value = Process.spawn ~name ~color "docker" args in - let run = Process.check in - {value; run} + Process.spawn ~name ~color "docker" args -let tag ?(tag = "latest") docker_registry = - let tezt_cloud = Lazy.force Env.tezt_cloud in +let tag ?image_name ?alias ?(tag = "latest") ~registry_uri () = + let alias = Option.value ~default:Env.dockerfile_alias alias in + let image_name = Option.value ~default:alias image_name in let args = [ "tag"; - Format.asprintf "%s:%s" tezt_cloud tag; - Format.asprintf "%s/%s:%s" docker_registry tezt_cloud tag; + Format.asprintf "%s:%s" image_name tag; + Format.asprintf "%s/%s:%s" registry_uri image_name tag; ] in - let value = Process.spawn ~name ~color "docker" args in - let run = Process.check in - {value; run} + Process.spawn ~name ~color "docker" args -let push ?(tag = "latest") docker_registry = - let tezt_cloud = Lazy.force Env.tezt_cloud in - let args = - ["push"; Format.asprintf "%s/%s:%s" docker_registry tezt_cloud tag] - in - let value = Process.spawn ~name ~color "docker" args in - let run = Process.check in - {value; run} +let push ?image_name ?alias ?(tag = "latest") ~registry_uri () = + let alias = Option.value ~default:Env.dockerfile_alias alias in + let image_name = Option.value ~default:alias image_name in + let args = ["push"; Format.asprintf "%s/%s:%s" registry_uri image_name tag] in + Process.spawn ~name ~color "docker" args -let pull ?(tag = "latest") docker_registry = - let tezt_cloud = Lazy.force Env.tezt_cloud in - let args = - ["pull"; Format.asprintf "%s/%s:%s" docker_registry tezt_cloud tag] - in - let value = Process.spawn ~name ~color "docker" args in - let run = Process.check in - {value; run} +let pull ?image_name ?alias ?(tag = "latest") ~registry_uri () = + let alias = Option.value ~default:Env.dockerfile_alias alias in + let image_name = Option.value ~default:alias image_name in + let args = ["pull"; Format.asprintf "%s/%s:%s" registry_uri image_name tag] in + Process.spawn ~name ~color "docker" args let run ?(rm = false) ?name ?network ?publish_ports image args = let publish_ports = @@ -67,40 +59,28 @@ let run ?(rm = false) ?name ?network ?publish_ports image args = in let name = match name with None -> [] | Some name -> ["--name"; name] in let rm = if rm then ["--rm"] else [] in - let value = - Process.spawn - ~color - "docker" - (["run"] @ rm @ name @ network @ publish_ports - @ [Format.asprintf "%s" image] - @ args) - in - let run = Process.check in - {value; run} + (* [init] can be used to ensure signals are forwarded properly to + the entrypoint run by the container. *) + Process.spawn + ~color + "docker" + (["run"] @ rm @ name @ network @ publish_ports + @ [Format.asprintf "%s" image] + @ args) -let kill container_name = - let value = Process.spawn ~color "docker" ["kill"; container_name] in - let run = Process.check in - {value; run} +let kill container_name = Process.spawn ~color "docker" ["kill"; container_name] -let rm container_name = - let value = Process.spawn ~color "docker" ["rm"; container_name] in - let run = Process.check in - {value; run} +let rm container_name = Process.spawn ~color "docker" ["rm"; container_name] let cp container_name ~kind ~source ~destination = - let value = - match kind with - | `From_host -> - Process.spawn - ~color - "docker" - ["cp"; source; Format.asprintf "%s:%s" container_name destination] - | `To_host -> - Process.spawn - ~color - "docker" - ["cp"; Format.asprintf "%s:%s" container_name source; destination] - in - let run = Process.check in - {value; run} + match kind with + | `From_host -> + Process.spawn + ~color + "docker" + ["cp"; source; Format.asprintf "%s:%s" container_name destination] + | `To_host -> + Process.spawn + ~color + "docker" + ["cp"; Format.asprintf "%s:%s" container_name source; destination] diff --git a/tezt/lib_cloud/docker.mli b/tezt/lib_cloud/docker.mli index d26c5cb8265a9761d4ef1ea153830ced52f7506b..693817cdf19667a7e562e3ffd81ff29ca514fb40 100644 --- a/tezt/lib_cloud/docker.mli +++ b/tezt/lib_cloud/docker.mli @@ -9,20 +9,40 @@ (** [build] is an alias for [docker build]. *) val build : + ?image_name:string -> + ?alias:string -> ?tag:string -> ?dockerfile:string -> args:(string * string) list -> unit -> - (Process.t, unit) runnable + Process.t (** [tag] is an alias for [docker tag]. *) -val tag : ?tag:string -> string -> (Process.t, unit) runnable +val tag : + ?image_name:string -> + ?alias:string -> + ?tag:string -> + registry_uri:string -> + unit -> + Process.t (** [push] is an alias for [docker push]. *) -val push : ?tag:string -> string -> (Process.t, unit) runnable +val push : + ?image_name:string -> + ?alias:string -> + ?tag:string -> + registry_uri:string -> + unit -> + Process.t (** [pull] is an alias for [docker pull]. *) -val pull : ?tag:string -> string -> (Process.t, unit) runnable +val pull : + ?image_name:string -> + ?alias:string -> + ?tag:string -> + registry_uri:string -> + unit -> + Process.t (** [run] is an alias for [docker run]. *) val run : @@ -32,13 +52,13 @@ val run : ?publish_ports:string * string * string * string -> string -> string list -> - (Process.t, unit) runnable + Process.t (** [kill] is an alias for [docker kill]. *) -val kill : string -> (Process.t, unit) runnable +val kill : string -> Process.t (** [rm] is an alias for [docker rm]. *) -val rm : string -> (Process.t, unit) runnable +val rm : string -> Process.t (** [cp] is an alias for [docker cp]. *) val cp : @@ -46,4 +66,4 @@ val cp : kind:[`From_host | `To_host] -> source:string -> destination:string -> - (Process.t, unit) runnable + Process.t diff --git a/tezt/lib_cloud/dockerfiles/octez.Dockerfile b/tezt/lib_cloud/dockerfiles/octez.Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..34a77a8040fc9e8839f0030722a2ea8acb8ff85f --- /dev/null +++ b/tezt/lib_cloud/dockerfiles/octez.Dockerfile @@ -0,0 +1,22 @@ +# Start from last known release +FROM tezos/tezos:latest + +# Set back the user for executing the following commands +USER root + +# SSH key that will be used for the SSH server +ARG SSH_PUBLIC_KEY + +# This is extracted from the link below +# https://dev.to/yakovlev_alexey/running-ssh-in-an-alpine-docker-container-3lop +RUN mkdir -p /root/.ssh \ + && chmod 0700 /root/.ssh \ + && echo "$SSH_PUBLIC_KEY" > /root/.ssh/authorized_keys \ + && apk add openrc openssh \ + && ssh-keygen -A \ + && echo -e "PasswordAuthentication no" >> /etc/ssh/sshd_config \ + && mkdir -p /run/openrc \ + && touch /run/openrc/softlevel + +CMD ["-D", "-p", "30000", "-e"] +ENTRYPOINT ["/usr/sbin/sshd"] diff --git a/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile b/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile index 3cb70ced47918f79f6e47a1405ef61d242795c28..cf17cc89dc7a4d930857420d8033e6fc74899094 100644 --- a/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile +++ b/tezt/lib_cloud/dockerfiles/saroupille.Dockerfile @@ -11,6 +11,8 @@ RUN apt-get update && apt-get install -y \ openssh-server \ # Tezos dependencies libgmp-dev curl libev-dev libhidapi-dev \ + # With the proxy mode we may want to run docker inside docker + docker.io screen \ # Necessary certificates for mirages dependencies ca-certificates \ # DL3015: Use --no-install-recommends diff --git a/tezt/lib_cloud/dune b/tezt/lib_cloud/dune index 8c5855675b5b582416a39e0aafbcb5d62f84a8bb..ecc8d09f296a8f0e6f85aa65ed350ae82fce1202 100644 --- a/tezt/lib_cloud/dune +++ b/tezt/lib_cloud/dune @@ -2,16 +2,14 @@ ; Edit file manifest/main.ml instead. (library - (name tezt_cloud) - (package tezt-cloud) + (name tezt_tezos_tezt_cloud) + (public_name tezt-tezos.tezt-cloud) (libraries tezt - tezt-tezos - tezt-tezos.tezt-performance-regression) + tezt-tezos.tezt-performance-regression + octez-libs.data-encoding) (flags (:standard) -open Tezt -open Tezt.Base - -open Tezt_tezos - -open Tezt_tezos.Runnable.Syntax -open Tezt_tezos_tezt_performance_regression)) diff --git a/tezt/lib_cloud/env.ml b/tezt/lib_cloud/env.ml index b396286cf59b8bc16f83d27d2bffdfc314b0f609..55a69be711bde76cdc576aebb7bb04614318f33c 100644 --- a/tezt/lib_cloud/env.ml +++ b/tezt/lib_cloud/env.ml @@ -5,47 +5,162 @@ (* *) (*****************************************************************************) +type docker_image = Gcp of {alias : string} | Octez_latest_release + let tezt_cloud = - (* This is a lazy value to be sure that this is evaluated only inside a Tezt test. *) - Lazy.from_fun @@ fun () -> - match Sys.getenv_opt "TEZT_CLOUD" with - | None -> - Test.fail - "The environment variable 'TEZT_CLOUD' is not defined. See README for \ - more information why this variable must be defined." - | Some value -> value - -let ssh_private_key = - Lazy.from_fun (fun () -> - let home = Sys.getenv "HOME" in - let tezt_cloud = Lazy.force tezt_cloud in - home // ".ssh" // Format.asprintf "%s-tf" tezt_cloud) - -let ssh_public_key = - Lazy.from_fun (fun () -> - let ssh_key = Lazy.force ssh_private_key in - Format.asprintf "%s.pub" ssh_key) - -let dockerfile = - Lazy.from_fun (fun () -> - let tezt_cloud = Lazy.force tezt_cloud in - let basename = Option.value ~default:tezt_cloud Cli.dockerfile in - Path.docker // Format.asprintf "%s.Dockerfile" basename) - -let docker_registry = - Lazy.from_fun (fun () -> - let tezt_cloud = Lazy.force tezt_cloud in - Format.asprintf "%s-docker-registry" tezt_cloud) - -let custom_docker_image ?docker_image_name ~project_id () = - let artifact_registry = "europe-west1-docker.pkg.dev" in - let docker_registry = Lazy.force docker_registry in - let docker_image_name = - Option.value ~default:(Lazy.force tezt_cloud) docker_image_name - in - Format.asprintf - "%s/%s/%s/%s" - artifact_registry - project_id - docker_registry - docker_image_name + match Cli.tezt_cloud with + | Some tezt_cloud -> tezt_cloud + | None -> ( + (* This is a lazy value to be sure that this is evaluated only inside a Tezt test. *) + match Sys.getenv_opt "TEZT_CLOUD" with None -> "" | Some value -> value) + +let ssh_private_key_filename ?(home = Sys.getenv "HOME") () = + home // ".ssh" // Format.asprintf "%s-tf" tezt_cloud + +let ssh_public_key_filename ?home () = + let ssh_key = ssh_private_key_filename ?home () in + Format.asprintf "%s.pub" ssh_key + +let docker_registry = Format.asprintf "%s-docker-registry" tezt_cloud + +let mode = + match (Cli.localhost, Cli.proxy) with + | true, true -> `Orchestrator + | true, false -> `Localhost + | false, true -> `Host + | false, false -> `Cloud + +let prometheus = Cli.prometheus + +let prometheus_export = Cli.prometheus_export + +let prometheus_port = Cli.prometheus_port + +let prometheus_snapshot_filename = Cli.prometheus_snapshot_filename + +let prometheus_scrape_interval = Cli.prometheus_scrape_interval + +let grafana = Cli.grafana + +let website = Cli.website + +let website_port = Cli.website_port + +let destroy = Cli.destroy + +let monitoring = Cli.monitoring + +let keep_alive = Cli.keep_alive + +let vms = Cli.vms + +let vm_base_port = Cli.vm_base_port + +let ports_per_vm = Cli.ports_per_vm + +let machine_type = Cli.machine_type + +let max_run_duration = Cli.max_run_duration + +let no_max_run_duration = Cli.no_max_run_duration + +let dns_domain = Cli.dns_domain + +let dns = Cli.dns + +let docker_image = + (* In localhost mode, we don't want to interact with GCP. The image is taken + locally. *) + match Cli.dockerfile_alias with + | None -> Gcp {alias = tezt_cloud} + | Some alias -> Gcp {alias} + +let dockerfile_alias = Option.value ~default:tezt_cloud Cli.dockerfile_alias + +let dockerfile = Path.dockerfile ~alias:dockerfile_alias + +let project_id = Gcloud.project_id + +let init () = + if tezt_cloud = "" then + Test.fail + "The tezt-cloud value should be set. Either via the CLI or via the \ + environement variable 'TEZT_CLOUD'" ; + match mode with + | `Localhost | `Orchestrator -> Lwt.return_unit + | `Host | `Cloud -> + let tezt_cloud = tezt_cloud in + let* project_id = project_id () in + Log.info "Initializing docker registry..." ; + let* () = Terraform.Docker_registry.init () in + Log.info "Deploying docker registry..." ; + let* () = Terraform.Docker_registry.deploy ~tezt_cloud ~project_id in + Lwt.return_unit + +(* Even though we could get this information locally, it is interesting to fetch + it through terraform to get the correct value if a scenario was launch with + different parameters. *) +let hostname = + let hostname = ref "" in + fun () -> + if !hostname = "" then ( + let tezt_cloud = tezt_cloud in + let* hostname' = Terraform.Docker_registry.get_hostname ~tezt_cloud in + hostname := hostname' ; + Log.info "Authenticate docker hostname..." ; + let* () = Gcloud.auth_configure_docker ~hostname:!hostname in + return !hostname) + else Lwt.return !hostname + +let zone = + let zone = ref "" in + fun () -> + (* To ensure the docker registry is deployed. *) + let* zone' = Terraform.VM.zone () in + zone := zone' ; + Lwt.return !zone + +let registry_uri () = + let* hostname = hostname () in + let* project_id = project_id () in + let uri = Format.asprintf "%s/%s/%s" hostname project_id docker_registry in + Lwt.return uri + +let uri_of_docker_image docker_image = + match (docker_image, mode) with + | Gcp {alias}, (`Cloud | `Host | `Orchestrator) -> + let* registry_uri = registry_uri () in + Lwt.return (Format.asprintf "%s/%s" registry_uri alias) + | Gcp {alias}, `Localhost -> Lwt.return alias + | Octez_latest_release, (`Cloud | `Host | `Orchestrator) -> + let* registry_uri = registry_uri () in + Lwt.return (Format.asprintf "%s/octez" registry_uri) + | Octez_latest_release, `Localhost -> Lwt.return "octez" + +let rec wait_process ?(sleep = 4) ~is_ready ~run () = + let process = run () in + let* status = Process.wait process in + match status with + | Unix.WEXITED 0 -> + let* output = Process.check_and_read_stdout process in + if is_ready output then Lwt.return output + else ( + Log.info + "Process '%s' is not ready. Let's wait %d seconds" + (Process.name process) + sleep ; + let* () = Lwt_unix.sleep (float_of_int sleep) in + wait_process ~sleep ~is_ready ~run ()) + | _ -> + Log.info + "Process '%s' failed. Let's wait %d seconds" + (Process.name process) + sleep ; + let* () = Lwt_unix.sleep (float_of_int sleep) in + wait_process ~sleep ~is_ready ~run () + +let run_command ?cmd_wrapper cmd args = + match cmd_wrapper with + | None -> Process.spawn cmd args + | Some cmd_wrapper -> + Process.spawn cmd_wrapper.Gcloud.cmd (cmd_wrapper.args @ [cmd] @ args) diff --git a/tezt/lib_cloud/env.mli b/tezt/lib_cloud/env.mli index da0405ead206abc0fbc39066ac2f55f5a1bdd171..a0acc73c3c114e8bd2839cc7f5cefac6bb3971e5 100644 --- a/tezt/lib_cloud/env.mli +++ b/tezt/lib_cloud/env.mli @@ -5,25 +5,94 @@ (* *) (*****************************************************************************) -(** This module contains values that depend on environment - variables. Those values are lazy so that they are evaluated within - a test and fail if one environment variable is missing. *) +(* This module aims to encapsulate static values from the CLI and several + functions that are used by the library. *) -(** Value of the environment variable [TEZT_CLOUD_BASENAME]. *) -val tezt_cloud : string Lazy.t +type docker_image = Gcp of {alias : string} | Octez_latest_release -(** Path to the ssh private key that will be used with the docker - image. Depends on [workspace]. *) -val ssh_private_key : string Lazy.t +val tezt_cloud : string -(** Path to the ssh public key that will be used with the docker - image. Depends on [workspace]. *) -val ssh_public_key : string Lazy.t +(** -(** Path to the docker image that will be used when running - VMs. Depends on [workspace]. *) -val dockerfile : string Lazy.t + - [Localhost]: Agents and the orchestrator are run on the host machine -(** URI where to find the custom docker image based on the [tezt_cloud] value. *) -val custom_docker_image : - ?docker_image_name:string -> project_id:string -> unit -> string + - [Cloud]: The orchestrator is on the host machine. The agents are run onto + the cloud + + - [`Orchestrator]: The orchestrator is run on a VM. This is mode used by the + proxy from the orchestrator point of view. + + - [`Host]: This mode is run by the host machine that initializes the + orchestrator running an a VM. +*) +val mode : [`Localhost | `Cloud | `Orchestrator | `Host] + +val prometheus : bool + +val prometheus_port : int + +val prometheus_export : bool + +val prometheus_snapshot_filename : string option + +val prometheus_scrape_interval : int + +val grafana : bool + +val website : bool + +val website_port : int + +val destroy : bool + +val monitoring : bool + +val keep_alive : bool + +val ssh_private_key_filename : ?home:string -> unit -> string + +val ssh_public_key_filename : ?home:string -> unit -> string + +val dockerfile_alias : string + +val dockerfile : string + +val docker_registry : string + +val docker_image : docker_image + +val vms : int option + +val vm_base_port : int + +val ports_per_vm : int + +val machine_type : string + +val max_run_duration : int + +val no_max_run_duration : bool + +val dns_domain : string option + +val dns : bool + +val init : unit -> unit Lwt.t + +val project_id : unit -> string Lwt.t + +val registry_uri : unit -> string Lwt.t + +val uri_of_docker_image : docker_image -> string Lwt.t + +val zone : unit -> string Lwt.t + +val wait_process : + ?sleep:int -> + is_ready:(string -> bool) -> + run:(unit -> Process.t) -> + unit -> + string Lwt.t + +val run_command : + ?cmd_wrapper:Gcloud.cmd_wrapper -> string -> string list -> Process.t diff --git a/tezt/lib_cloud/gcloud.ml b/tezt/lib_cloud/gcloud.ml index 86d537782d838539a40d178a2be22edb546b4b4c..be45ddfaae93c38e6580384a815133324420dcdc 100644 --- a/tezt/lib_cloud/gcloud.ml +++ b/tezt/lib_cloud/gcloud.ml @@ -32,27 +32,27 @@ let project_id = Lwt.return !project_id) else Lwt.return !project_id -let compute_ssh ~zone ~vm_name cmd args = - let ssh_private_key = Lazy.force Env.ssh_private_key in - let value = - Process.spawn - ~name - ~color - "gcloud" - ([ - "compute"; - "ssh"; - "--ssh-key-file"; - ssh_private_key; - vm_name; - "--zone"; - zone; - "--"; - ] - @ [cmd] @ args) - in - let run = Process.check_and_read_stdout in - {value; run} +type cmd_wrapper = {cmd : string; args : string list} + +let cmd_wrapper ~zone ~vm_name ~ssh_private_key_filename = + { + cmd = "gcloud"; + args = + [ + "compute"; + "ssh"; + "--ssh-key-file"; + ssh_private_key_filename; + vm_name; + "--zone"; + zone; + "--"; + ]; + } + +let compute_ssh ~zone ~vm_name ~ssh_private_key_filename cmd args = + let wrapper = cmd_wrapper ~zone ~vm_name ~ssh_private_key_filename in + Process.spawn ~name ~color wrapper.cmd (wrapper.args @ [cmd] @ args) let get_ip_address_from_name ~zone name = let* output = @@ -79,3 +79,100 @@ let list_vms ~prefix = ["compute"; "instances"; "list"; "--filter"; filter] in Lwt.return (String.trim output) + +module DNS = struct + let create_zone ~domain ~zone () = + let dns_name = Format.asprintf "%s.%s" zone domain in + let description = + Format.asprintf "Managed zone for tezt-cloud for %s project" zone + in + Process.run + "gcloud" + [ + "dns"; + "managed-zones"; + "create"; + zone; + "--dns-name"; + dns_name; + "--description"; + description; + ] + + let describe ~zone () = + Process.run_and_read_stdout + "gcloud" + ["dns"; "managed-zones"; "describe"; zone] + + module Transaction = struct + let start ~zone () = + Process.run + "gcloud" + ["dns"; "record-sets"; "transaction"; "start"; "--zone"; zone] + + let add ?(ttl = 300) ?(typ = "A") ~domain ~zone ~ip () = + let ttl = ["--ttl"; Format.asprintf "%d" ttl] in + let typ = ["--type"; Format.asprintf "%s" typ] in + let domain = ["--name"; domain] in + let zone = ["--zone"; zone] in + Process.run + "gcloud" + (["dns"; "record-sets"; "transaction"; "add"] + @ zone @ ttl @ typ @ domain @ [ip]) + + let remove ?(ttl = 300) ?(typ = "A") ~domain ~zone ~ip () = + let ttl = ["--ttl"; Format.asprintf "%d" ttl] in + let typ = ["--type"; Format.asprintf "%s" typ] in + let domain = ["--name"; domain] in + let zone = ["--zone"; zone] in + Process.run + "gcloud" + (["dns"; "record-sets"; "transaction"; "remove"] + @ zone @ ttl @ typ @ domain @ [ip]) + + let execute ~zone () = + Process.run + "gcloud" + ["dns"; "record-sets"; "transaction"; "execute"; "--zone"; zone] + + let abort ~zone () = + Process.run + "gcloud" + ["dns"; "record-sets"; "transaction"; "abort"; "--zone"; zone] + end + + let get_domain ~tezt_cloud ~zone = + let* output = describe ~zone () in + let line = + String.trim output |> String.split_on_char '\n' + |> List.find_opt (String.starts_with ~prefix:"dnsName:") + in + match line with + | None -> Test.fail "Unable to find a managed zone. Have you create it?" + | Some line -> + Lwt.return + (Format.asprintf + "%s.%s" + tezt_cloud + (String.split_on_char ' ' line |> Fun.flip List.nth 1)) + + let add ~tezt_cloud ~zone ~ip = + let* domain = get_domain ~tezt_cloud ~zone in + Lwt.catch + (fun () -> + let* () = Transaction.start ~zone () in + let* () = Transaction.add ~domain ~zone ~ip () in + let* () = Transaction.execute ~zone () in + unit) + (fun _ -> Transaction.abort ~zone ()) + + let remove ~tezt_cloud ~zone ~ip = + let* domain = get_domain ~tezt_cloud ~zone in + Lwt.catch + (fun () -> + let* () = Transaction.start ~zone () in + let* () = Transaction.remove ~domain ~zone ~ip () in + let* () = Transaction.execute ~zone () in + unit) + (fun _ -> Transaction.abort ~zone ()) +end diff --git a/tezt/lib_cloud/gcloud.mli b/tezt/lib_cloud/gcloud.mli index ed6455b59e9e089cada9ef1052965c0d8999e280..7b04b2a91f51e73491bb4cabefa6977f018c84c0 100644 --- a/tezt/lib_cloud/gcloud.mli +++ b/tezt/lib_cloud/gcloud.mli @@ -22,12 +22,36 @@ val project_id : unit -> string Lwt.t val compute_ssh : zone:string -> vm_name:string -> + ssh_private_key_filename:string -> string -> string list -> - (Process.t, string) runnable + Process.t + +type cmd_wrapper = {cmd : string; args : string list} + +(** [cmd_wrapper ~zone ~vm_name cmd args] is the wrapper to run a command on the + vm [vm_name]. + *) +val cmd_wrapper : + zone:string -> + vm_name:string -> + ssh_private_key_filename:string -> + cmd_wrapper (** [get_ip_address_from_name ~zone name] allows to get the external IP address of a VM from its name [name]. *) val get_ip_address_from_name : zone:string -> string -> string Lwt.t val list_vms : prefix:string -> string Lwt.t + +module DNS : sig + val create_zone : domain:string -> zone:string -> unit -> unit Lwt.t + + val describe : zone:string -> unit -> string Lwt.t + + val get_domain : tezt_cloud:string -> zone:string -> string Lwt.t + + val add : tezt_cloud:string -> zone:string -> ip:string -> unit Lwt.t + + val remove : tezt_cloud:string -> zone:string -> ip:string -> unit Lwt.t +end diff --git a/tezt/lib_cloud/grafana.ml b/tezt/lib_cloud/grafana.ml index 39b0056d1a08752b593d960c2ff18a41386d4ec2..39cbfed6aaee761f291ce4f767fb62f299b7269b 100644 --- a/tezt/lib_cloud/grafana.ml +++ b/tezt/lib_cloud/grafana.ml @@ -7,12 +7,52 @@ include Tezt_tezos_tezt_performance_regression.Grafana -type t = {provisioning_file : string; dashboard_directory : string} +type t = { + provisioning_directory : string; + dashboard_directory : string; + password : string; +} -let provisioning_file () = +let generate_password () = "saucisse" + +let generate_admin_api_key password = + let cmd = "curl" in + let args = + [ + "-X"; + "POST"; + "-H"; + "Content-Type: application/json"; + "-d"; + "\n\ + \ {\n\ + \ \"name\": \"admin_api_key\",\n\ + \ \"role\": \"Admin\"\n\ + \ }"; + "-u"; + Format.asprintf "admin:%s" password; + "http://localhost:3000/api/auth/keys"; + ] + in + let* output = Process.run_and_read_stdout cmd args in + let json = JSON.parse ~origin:"Grafana.generate_admin_api_key" output in + let key = JSON.(json |-> "key" |> as_string) in + Lwt.return key + +let configuration admin_api_key : config = + { + url = Uri.of_string "http://localhost:3000"; + api_token = Some admin_api_key; + data_source = "Prometheus"; + timeout = 2.0; + } + +let provisioning_directory () = + let provisioning_directory = + Filename.get_temp_dir_name () // "grafana" // "provisioning" + in let provisioning_file = - Filename.get_temp_dir_name () - // "grafana" // "provisioning" // "provisioning.yml" + provisioning_directory // "datasources" // "datasource.yml" in let* () = Process.run "mkdir" ["-p"; provisioning_file |> Filename.dirname] in let content = @@ -23,26 +63,30 @@ datasources: - name: Prometheus type: prometheus access: proxy - url: http://prometheus:9090 + url: http://localhost:9090 isDefault: true - -providers: - - name: 'default' - orgId: 1 - folder: '' - type: file - disableDeletion: true - options: - path: /var/lib/grafana/dashboards |} in with_open_out provisioning_file (fun oc -> Stdlib.seek_out oc 0 ; output_string oc content) ; - Lwt.return provisioning_file + Lwt.return provisioning_directory let shutdown _t = Process.run "docker" ["kill"; "grafana"] +let dashboards_filepaths () = + let path = + if Env.mode = `Orchestrator then + Filename.get_temp_dir_name () // "grafana" // "dashboards" + else Path.grafana_dashboards + in + let* output = Process.run_and_read_stdout "ls" [path] in + let basenames = + String.trim output |> String.split_on_char '\n' + |> List.filter (fun file -> String.ends_with ~suffix:".json" file) + in + List.map (fun basename -> path // basename) basenames |> Lwt.return + let run () = let cmd = "docker" in let* () = @@ -54,7 +98,11 @@ let run () = let* () = Process.run "mkdir" ["-p"; dashboard_directory |> Filename.dirname] in - let* provisioning_file = provisioning_file () in + let* provisioning_directory = provisioning_directory () in + (* We generate a password to use admin features. This is not completely + secured but this should prevent easy attacks if the grafana port is opened. *) + let password = generate_password () in + Log.info "Grafana admin password: %s" password ; let args = [ "run"; @@ -64,16 +112,14 @@ let run () = "grafana"; "--network"; "host"; - "-p"; - "3000:3000"; "-e"; "GF_AUTH_ANONYMOUS_ENABLED=true"; "-e"; "GF_AUTH_ANONYMOUS_ORG_ROLE=Viewer"; + "-e"; + Format.asprintf "GF_SECURITY_ADMIN_PASSWORD=%s" password; "-v"; - Format.asprintf - "%s:/etc/grafana/provisioning" - (Filename.dirname provisioning_file); + Format.asprintf "%s:/etc/grafana/provisioning" provisioning_directory; "-v"; Format.asprintf "%s:/var/lib/grafana/dashboards" dashboard_directory; "grafana/grafana"; @@ -88,4 +134,37 @@ let run () = let* () = shutdown () in Process.run cmd args in - Lwt.return {provisioning_file; dashboard_directory} + let is_ready output = String.trim output = "200" in + let run () = + Process.spawn + "curl" + [ + "-s"; + "-o"; + "/dev/null"; + "-w"; + "%{http_code}"; + "http://localhost:3000/api/health"; + ] + in + let* _ = Env.wait_process ~is_ready ~run () in + let* admin_api_key = generate_admin_api_key password in + let configuration = configuration admin_api_key in + let* dashboards_filepaths = dashboards_filepaths () in + let dashboard dashboard_filepath = + read_file dashboard_filepath + |> Format.asprintf "{\"dashboard\": %s, \"overwrite\": true}" + in + let rec loop = function + | [] -> Lwt.return_unit + | name :: l -> + let* () = + update_dashboard_from_json + configuration + ~json:(dashboard name) + ~uid:name + in + loop l + in + let* () = loop dashboards_filepaths in + Lwt.return {provisioning_directory; dashboard_directory; password} diff --git a/tezt/lib_cloud/grafana.mli b/tezt/lib_cloud/grafana.mli index ee3b5694eca909cc8e6a2a867ea059c964e598a7..84628e457df2da76a3fc4f2ec8f316619c860a85 100644 --- a/tezt/lib_cloud/grafana.mli +++ b/tezt/lib_cloud/grafana.mli @@ -12,3 +12,5 @@ type t val run : unit -> t Lwt.t val shutdown : t -> unit Lwt.t + +val dashboards_filepaths : unit -> string list Lwt.t diff --git a/tezt/lib_cloud/grafana/dashboards/README.md b/tezt/lib_cloud/grafana/dashboards/README.md new file mode 100644 index 0000000000000000000000000000000000000000..83a4de69b93ea954af7bd5f3fb04b8e47999a390 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/README.md @@ -0,0 +1,12 @@ +# Grafana dashboards + +Those dashboards where generated via the +[grafazos](https://gitlab.com/nomadic-labs/grafazos/-/tree/master?ref_type=heads) +project using the following command: + +``` +$ make all NODE_INSTANCE_LABEL=app +``` + +The result where copied in this directory. Feel free to update those +dashboards in consequence. diff --git a/tezt/lib_cloud/grafana/dashboards/dal-basic.json b/tezt/lib_cloud/grafana/dashboards/dal-basic.json new file mode 100644 index 0000000000000000000000000000000000000000..4bbaa226d320bbd8ad30d4d7b71c933c945bcc1c --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/dal-basic.json @@ -0,0 +1,1951 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "DAL node's core stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Finalized L1 blocks": "green", + "Seen L1 heads": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_node_new_layer1_head{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 1, + "legendFormat": "Seen L1 heads", + "refId": "A" + }, + { + "expr": "dal_node_layer1_block_finalized{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 1, + "legendFormat": "Finalized L1 blocks", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Layer 1 heads & finalized blocks seen by the DAL node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Finalized L1 blocks' rounds": "green", + "Seen L1 heads' rounds": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_node_new_layer1_head_round{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 1, + "legendFormat": "Seen L1 heads' rounds", + "refId": "A" + }, + { + "expr": "dal_node_layer1_block_finalized_round{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 1, + "legendFormat": "Finalized L1 blocks' rounds", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Rounds of layer 1 heads & finalized blocks seen by the DAL node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Stored shards": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 5, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(dal_node_number_of_stored_shards{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Stored shards", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "The shards stored by this node (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Number of attested slots": "green", + "Number of slots waiting for attestation": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 9 + }, + "id": 6, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(dal_node_slots_waiting_for_attestaion{app=\"$node_instance\", slot_index=~\"$slot_index\"})", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Number of slots waiting for attestation", + "refId": "A" + }, + { + "expr": "sum(dal_node_slots_attested{app=\"$node_instance\", slot_index=~\"$slot_index\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Number of attested slots", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Number of slots waiting for attesatation and of attested slots", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 9 + }, + "id": 7, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_node_slots_waiting_for_attestaion{app=\"$node_instance\", slot_index=~\"$slot_index\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Slot index: {{ slot_index }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Indexes of slots waiting for attestation", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 9 + }, + "id": 8, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_node_slots_attested{app=\"$node_instance\", slot_index=~\"$slot_index\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Slot index: {{ slot_index }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Indexes of attested slots", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 24, + "x": 0, + "y": 17 + }, + "id": 9, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Gossipsub worker's stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 10, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_scores_of_peers{app=\"$node_instance\", peer=\"$peer\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "{{ peer }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Score of the peers connected to this node", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 11, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_count_peers_per_topic{app=\"$node_instance\", slot_index=~\"$slot_index\", pkh=\"$pkh\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Topic: {{ slot_index }}, {{ pkh }}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Number of peers in the mesh for each subscribed topic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Node's topics": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 25 + }, + "id": 12, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_count_topics{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node's topics", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "The number of topics the node is subscribed to", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "All connections": "green", + "Bootstrap connections": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 25 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_count_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "All connections", + "refId": "A" + }, + { + "expr": "dal_gs_count_bootstrap_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bootstrap connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "The number of (bootstrap) connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Application output stream": "yellow", + "Input stream": "green", + "P2P output stream": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 25 + }, + "id": 14, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "dal_gs_input_events_stream_length{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Input stream", + "refId": "A" + }, + { + "expr": "dal_gs_p2p_output_stream_length{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "P2P output stream", + "refId": "B" + }, + { + "expr": "dal_gs_app_output_stream_length{app=\"$node_instance\"}", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Application output stream", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "The size of different worker streams", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Received & invalid": "red", + "Received & unchecked validity": "orange", + "Received & valid": "green", + "Sent": "blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 33 + }, + "id": 15, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(dal_gs_count_received_valid_messages{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Received & valid", + "refId": "A" + }, + { + "expr": "deriv(dal_gs_count_received_invalid_messages{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Received & invalid", + "refId": "B" + }, + { + "expr": "deriv(dal_gs_count_received_unknown_validity_messages{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Received & unchecked validity", + "refId": "C" + }, + { + "expr": "deriv(dal_gs_count_sent_messages{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Average sent & received app messages (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Graft": "blue", + "IHave": "purple", + "IWant": "magenta", + "Prune": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 33 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(dal_gs_count_sent_grafts{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Graft", + "refId": "A" + }, + { + "expr": "deriv(dal_gs_count_sent_prunes{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Prune", + "refId": "B" + }, + { + "expr": "deriv(dal_gs_count_sent_ihaves{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "IHave", + "refId": "C" + }, + { + "expr": "deriv(dal_gs_count_sent_iwants{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "IWant", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Average sent control & metadata messages (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Graft": "blue", + "IHave": "purple", + "IWant": "magenta", + "Prune": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 33 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(dal_gs_count_received_grafts{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Graft", + "refId": "A" + }, + { + "expr": "deriv(dal_gs_count_received_prunes{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "Prune", + "refId": "B" + }, + { + "expr": "deriv(dal_gs_count_received_ihaves{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "IHave", + "refId": "C" + }, + { + "expr": "deriv(dal_gs_count_received_iwants{app=\"$node_instance\"}[1m])", + "format": "time_series", + "interval": "3s", + "intervalFactor": 2, + "legendFormat": "IWant", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Average received control & metadata messages (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 18, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "P2P stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Received data (KB)": "blue", + "Sent data (KB)": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 41 + }, + "id": 19, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "deriv(octez_p2p_io_scheduler_total_recv{app=\"$node_instance\"}[1m]) / 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Received data (KB)", + "refId": "A" + }, + { + "expr": "deriv(octez_p2p_io_scheduler_total_sent{app=\"$node_instance\"}[1m]) / 1000", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sent data (KB)", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Average data exchange (1-minute interval)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 41 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected peers": "light-yellow", + "Running peers": "light-green", + "Waiting to reconnect": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 49 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_peers_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected peers", + "refId": "A" + }, + { + "expr": "octez_p2p_peers_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running peers", + "refId": "B" + }, + { + "expr": "octez_p2p_points_greylisted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Waiting to reconnect", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P peers connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected points": "light-red", + "Running points": "light-green", + "Trusted points": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 49 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_points_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected points", + "refId": "A" + }, + { + "expr": "octez_p2p_points_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running points", + "refId": "B" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P points connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez", + "dal" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(dal_node_new_layer1_head,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "0", + "value": "0" + }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": true, + "label": "Slot index", + "multi": true, + "name": "slot_index", + "options": [ ], + "query": "label_values(dal_gs_count_peers_per_topic, slot_index)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Pkh", + "multi": false, + "name": "pkh", + "options": [ ], + "query": "label_values(dal_gs_count_peers_per_topic, pkh)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Peer", + "multi": false, + "name": "peer", + "options": [ ], + "query": "label_values(dal_gs_scores_of_peers, peer)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez DAL Node Dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/evm-node.json b/tezt/lib_cloud/grafana/dashboards/evm-node.json new file mode 100644 index 0000000000000000000000000000000000000000..601e0f94a6b69a63e35e0ab64fc1651c799a2338 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/evm-node.json @@ -0,0 +1,1093 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": null, + "links": [], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_info{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "{{commit_hash}}", + "range": false, + "refId": "A" + } + ], + "title": "Commit", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 3, + "y": 0 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_info{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "{{commit_date}}", + "range": false, + "refId": "A" + } + ], + "title": "Commit Date", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 9, + "y": 0 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_info{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "{{devmode}}", + "range": false, + "refId": "A" + } + ], + "title": "Devmode", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 12, + "y": 0 + }, + "id": 12, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_info{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "{{mode}}", + "range": false, + "refId": "A" + } + ], + "title": "Mode", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 15, + "y": 0 + }, + "id": 10, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(octez_evm_node_calls_count{app=\"$node_instance\"})", + "instant": true, + "legendFormat": "{{commit_hash}}", + "range": false, + "refId": "A" + } + ], + "title": "RPC Calls", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(octez_evm_node_calls_sum{app=\"$node_instance\"})/sum(octez_evm_node_calls_count{app=\"$node_instance\"})", + "instant": true, + "legendFormat": "{{commit_hash}}", + "range": false, + "refId": "A" + } + ], + "title": "RPC Average duration", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 2, + "x": 0, + "y": 3 + }, + "id": 5, + "options": { + "colorMode": "none", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "value_and_name", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_head{app=\"$node_instance\"}", + "instant": true, + "legendFormat": "Node head", + "range": false, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_confirmed_head{app=\"$node_instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Rollup node head", + "range": true, + "refId": "B" + } + ], + "title": "Head", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 11, + "x": 2, + "y": 3 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_confirmed_head{app=\"$node_instance\"}", + "instant": false, + "legendFormat": "Rollup-node's level", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_head{app=\"$node_instance\"}", + "hide": false, + "instant": false, + "legendFormat": "Node's level", + "range": true, + "refId": "B" + } + ], + "title": "Levels", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "greater", + "options": { + "value": 0 + } + }, + "fieldName": "Rollup-node's level" + } + ], + "match": "any", + "type": "include" + } + } + ], + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "description": "The average is computed over 15 seconds.", + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "yellow", + "mode": "shades", + "seriesBy": "last" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisGridShow": true, + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMin": 10, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 1, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "always", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "E" + }, + "properties": [ + { + "id": "displayName", + "value": "Observed time processed" + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Observed time processed" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 11, + "x": 13, + "y": 3 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "octez_evm_node_time_processed{app=\"$node_instance\"}", + "fullMetaSearch": false, + "hide": true, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "Processing Time", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_time_processed{app=\"$node_instance\"} offset 15s", + "hide": true, + "instant": false, + "legendFormat": "Offset Time Processed", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_head{app=\"$node_instance\"}", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_head{app=\"$node_instance\"} offset 15s", + "hide": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "__expr__", + "uid": "__expr__" + }, + "expression": "($A-$B)/($C-$D)", + "hide": false, + "refId": "E", + "type": "math" + } + ], + "title": "Average processing time", + "transformations": [ + { + "id": "filterByValue", + "options": { + "filters": [ + { + "config": { + "id": "greater", + "options": { + "value": 0 + } + }, + "fieldName": "E" + } + ], + "match": "any", + "type": "include" + } + } + ], + "type": "timeseries" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "decimals": 0, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 13, + "x": 0, + "y": 13 + }, + "id": 7, + "options": { + "displayMode": "gradient", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "disableTextWrap": false, + "editorMode": "builder", + "exemplar": true, + "expr": "octez_evm_node_block_process_time_histogram_bucket{app=\"$node_instance\"}", + "format": "heatmap", + "fullMetaSearch": false, + "includeNullMetadata": false, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Processing time distribution", + "type": "bargauge" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 21 + }, + "id": 13, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_tx_pool{kind=\"number_of_addresses\", app=\"$node_instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Transaction pool addresses count", + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 21 + }, + "id": 14, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "value", + "wideLayout": true + }, + "pluginVersion": "10.4.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "ddj6uqupldk3kd" + }, + "editorMode": "code", + "expr": "octez_evm_node_tx_pool{kind=\"number_of_transactions\", app=\"$node_instance\"}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Transaction pool transactions count", + "type": "stat" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { }, + "datasource": "Prometheus", + "definition": "label_values(octez_evm_node_info,app)", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(octez_evm_node_info,app)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-15m", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Etherlink", + "version": 0, + "weekStart": "" +} diff --git a/tezt/lib_cloud/grafana/dashboards/octez-basic.json b/tezt/lib_cloud/grafana/dashboards/octez-basic.json new file mode 100644 index 0000000000000000000000000000000000000000..9b254c0d67aeb9f90342b3e84da687758cce3570 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/octez-basic.json @@ -0,0 +1,5139 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Node stats", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bootstrapping" + }, + "1": { + "color": "green", + "index": 0, + "text": "Bootstrapped" + }, + "null": { + "color": "yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 3, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_is_bootstrapped{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bootstrap status", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Unsync" + }, + "1": { + "color": "green", + "index": 1, + "text": "Sync" + }, + "2": { + "color": "red", + "index": 2, + "text": "Stuck" + }, + "null": { + "color": "yellow", + "index": 3, + "text": "Unknow" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 4, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_synchronisation_status{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sync status", + "transparent": false, + "type": "stat" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ chain_name }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Chain name", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 8, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 11, + "y": 1 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_version{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Reflects the uptime of the monitoring of the job, not the uptime of the process.", + "format": "dtdhms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time()-(process_start_time_seconds{app=\"$node_instance\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "node uptime", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 7 + }, + "id": 9, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current head level", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current head level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 10 + }, + "id": 10, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ p2p_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "P2p version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 10 + }, + "id": 11, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ distributed_db_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Distributed db version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "value" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 13 + }, + "id": 12, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_savepoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current savepoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current savepoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 13 + }, + "id": 13, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_checkpoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current checkpoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current checkpoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 16 + }, + "id": 14, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_caboose_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current caboose", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current caboose level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 16 + }, + "id": 15, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_cycle{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current cycle", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current cycle", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 19 + }, + "id": 16, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trusted points", + "refId": "A" + } + ], + "thresholds": "", + "title": "Trusted points", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 19 + }, + "id": 17, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_connections_private{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private Connections", + "refId": "A" + } + ], + "thresholds": "", + "title": "Private Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "aliasColors": { + "Head level": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 4, + "y": 4 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head level", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head level history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Validation time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 14 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validation time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block validation time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 22 + }, + "id": 20, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"0\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consensus", + "refId": "A" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"1\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Vote", + "refId": "B" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"2\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anonymous", + "refId": "C" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"3\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Manager", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Gas consumed": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 30 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_consumed_gas{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Gas consumed", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Gas consumed history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Invalid blocks": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 10, + "x": 14, + "y": 1 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_invalid_blocks{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Invalid blocks", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Invalid blocks history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Round": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 10, + "x": 14, + "y": 10 + }, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Round", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Round history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Written block size": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 14, + "y": 19 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_written_block_size{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Written block size", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Last written block size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Merge time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 10, + "x": 14, + "y": 29 + }, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_merge_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Merge time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Store merge time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 39 + }, + "id": 26, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "P2P stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Branch delayed": "light-red", + "Branch refused": "light-orange", + "Outdated": "blue", + "Refused": "red", + "Unprocessed": "light-yellow", + "Validated": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_mempool_pending_validated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validated", + "refId": "A" + }, + { + "expr": "octez_mempool_pending_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Refused", + "refId": "B" + }, + { + "expr": "octez_mempool_pending_outdated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outdated", + "refId": "C" + }, + { + "expr": "octez_mempool_pending_branch_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch refused", + "refId": "D" + }, + { + "expr": "octez_mempool_pending_branch_delayed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch delayed", + "refId": "E" + }, + { + "expr": "octez_mempool_pending_unprocessed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unprocessed", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Mempool status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected peers": "light-yellow", + "Running peers": "light-green", + "Waiting to reconnect": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_peers_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected peers", + "refId": "A" + }, + { + "expr": "octez_p2p_peers_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running peers", + "refId": "B" + }, + { + "expr": "octez_p2p_points_greylisted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Waiting to reconnect", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P peers connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected points": "light-red", + "Running points": "light-green", + "Trusted points": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_points_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected points", + "refId": "A" + }, + { + "expr": "octez_p2p_points_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running points", + "refId": "B" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P points connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current head broadcasted": "light-green", + "current head received": "light-blue", + "current head sent": "light-green", + "get current head broadcasted": "green", + "get current head received": "blue", + "get current head sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 55 + }, + "id": 31, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head broadcasted", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head sent", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head received", + "refId": "D" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head sent", + "refId": "E" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head broadcasted", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current head messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current branch received": "light-blue", + "current branch sent": "light-green", + "get current branch received": "blue", + "get current branch sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 55 + }, + "id": 32, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block header received": "light-blue", + "block header sent": "light-green", + "get block headers received": "blue", + "get block headers sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 55 + }, + "id": 33, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB block headers messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get predecessor header received": "blue", + "get predecessor header sent": "green", + "predecessor header received": "light-blue", + "predecessor header sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 55 + }, + "id": 34, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB predecessor header messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get operations received": "blue", + "get operations sent": "green", + "operation received": "light-blue", + "operation sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 63 + }, + "id": 35, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get ops for blocks received": "blue", + "get ops for blocks sent": "green", + "ops for block received": "light-blue", + "ops for block sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 63 + }, + "id": 36, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations for blocks messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocols received": "blue", + "get protocols sent": "green", + "protocol received": "light-blue", + "protocol sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 63 + }, + "id": 37, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocols messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocol branch received": "blue", + "get protocol branch sent": "green", + "protocol branch received": "light-blue", + "protocol branch sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 63 + }, + "id": 38, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocol branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "checkpoint received": "light-blue", + "checkpoint sent": "light-green", + "get checkpoint received": "blue", + "get checkpoint sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 63 + }, + "id": 39, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB checkpoint messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "deactivate received": "light-blue", + "deactivate sent": "light-green", + "get deactivate received": "blue", + "get deactivate sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 63 + }, + "id": 40, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB deactivate messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 72 + }, + "id": 41, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Workers stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 42, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator request count", + "refId": "A" + }, + { + "expr": "octez_validator_block_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator completion count", + "refId": "B" + }, + { + "expr": "octez_validator_block_worker_request_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator error count", + "refId": "C" + }, + { + "expr": "octez_validator_chain_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator request count", + "refId": "D" + }, + { + "expr": "octez_validator_chain_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator completion count", + "refId": "E" + }, + { + "expr": "octez_validator_chain_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator error count", + "refId": "F" + }, + { + "expr": "octez_mempool_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator request count", + "refId": "G" + }, + { + "expr": "octez_mempool_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator completion count", + "refId": "H" + }, + { + "expr": "octez_mempool_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator error count", + "refId": "I" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validation workers requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers table", + "refId": "A" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers scheduler", + "refId": "B" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations table", + "refId": "C" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations scheduler", + "refId": "D" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation table", + "refId": "E" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation scheduler", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB Workers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 88 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Treatments", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 88 + }, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Completion", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "connections": "light-green", + "fetching canceled knew unknown head": "light-orange", + "fetching canceled new branch": "green", + "fetching canceled new known valid head": "light-yellow", + "invalid blocks": "light-red", + "invalid locator": "light-orange", + "new branch completed": "light-blue", + "new head completed": "blue", + "on no request count": "white", + "system error": "orange", + "too short locator": "brown", + "unavailable protocol": "light-red", + "unknown ancestor": "yellow", + "unknown error": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "id": 46, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_peer_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "connections", + "refId": "A" + }, + { + "expr": "octez_validator_peer_invalid_block{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid blocks", + "refId": "B" + }, + { + "expr": "octez_validator_peer_invalid_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid locator", + "refId": "C" + }, + { + "expr": "octez_validator_peer_new_branch_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new branch completed", + "refId": "D" + }, + { + "expr": "octez_validator_peer_new_head_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new head completed", + "refId": "E" + }, + { + "expr": "octez_validator_peer_on_no_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "on no request count", + "refId": "F" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_branch{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new branch", + "refId": "G" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_known_valid_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new known valid head", + "refId": "H" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_unknown_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled knew unknown head", + "refId": "I" + }, + { + "expr": "octez_validator_peer_system_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "system error", + "refId": "J" + }, + { + "expr": "octez_validator_peer_too_short_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "too short locator", + "refId": "K" + }, + { + "expr": "octez_validator_peer_unavailable_protocol{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unavailable protocol", + "refId": "L" + }, + { + "expr": "octez_validator_peer_unknown_ancestor{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown ancestor", + "refId": "M" + }, + { + "expr": "octez_validator_peer_unknown_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown error", + "refId": "N" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Peer validators", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 16, + "w": 8, + "x": 0, + "y": 97 + }, + "id": 47, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "RPC", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 97 + }, + "id": 48, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC calls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/version.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/config.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/network.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/workers.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/injection.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/protocols.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/private.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 97 + }, + "id": 50, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total calls", + "refId": "A" + } + ], + "title": "Total of RPC calls", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 97 + }, + "id": 51, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(octez_rpc_calls_count{endpoint=~\".*\",app=\"$node_instance\"}[1h]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "calls rate", + "refId": "A" + } + ], + "title": "RPC calls rate per hour", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 105 + }, + "id": 52, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average duration", + "refId": "A" + } + ], + "title": "Average of calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 105 + }, + "id": 53, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"} / octez_rpc_calls_count{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max average duration", + "refId": "A" + } + ], + "title": "Max of average calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 105 + }, + "id": 54, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max total duration", + "refId": "A" + } + ], + "title": "Max of total calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 105 + }, + "id": 55, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total duration", + "refId": "A" + } + ], + "title": "Total of RPC calls duration", + "transparent": false, + "type": "stat" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 114 + }, + "id": 56, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Miscellaneous", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Forced major collections": "light-blue", + "Heap compactions": "light-red", + "Major collections": "light-yellow", + "Minor collections": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 114 + }, + "id": 57, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_minor_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Minor collections", + "refId": "A" + }, + { + "expr": "ocaml_gc_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major collections", + "refId": "B" + }, + { + "expr": "ocaml_gc_forced_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Forced major collections", + "refId": "C" + }, + { + "expr": "ocaml_gc_compactions{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Heap compactions", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC maintenance operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Major heap": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 114 + }, + "id": 58, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_heap_words{app=\"$node_instance\"}* 8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major heap", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC major word sizes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "5s", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_version,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez basic dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/octez-compact.json b/tezt/lib_cloud/grafana/dashboards/octez-compact.json new file mode 100644 index 0000000000000000000000000000000000000000..ea52382d6611ceac876a43e51664473f803f0109 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/octez-compact.json @@ -0,0 +1,1941 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bootstrapping" + }, + "1": { + "color": "green", + "index": 0, + "text": "Bootstrapped" + }, + "null": { + "color": "yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 2, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_is_bootstrapped{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bootstrap status", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Unsync" + }, + "1": { + "color": "green", + "index": 1, + "text": "Sync" + }, + "2": { + "color": "red", + "index": 2, + "text": "Stuck" + }, + "null": { + "color": "yellow", + "index": 3, + "text": "Unknow" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 3, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_synchronisation_status{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sync status", + "transparent": false, + "type": "stat" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 6, + "y": 0 + }, + "id": 4, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ chain_name }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Chain name", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 10, + "y": 0 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 13, + "y": 0 + }, + "id": 6, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_version{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Reflects the uptime of the monitoring of the job, not the uptime of the process.", + "format": "dtdhms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 3 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time()-(process_start_time_seconds{app=\"$node_instance\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "node uptime", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 6 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current head level", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current head level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "columns": [ ], + "datasource": "Prometheus", + "gridPos": { + "h": 6, + "w": 4, + "x": 0, + "y": 9 + }, + "id": 9, + "links": [ ], + "styles": [ + { + "alias": "Time", + "pattern": "Time", + "type": "hidden" + } + ], + "targets": [ + { + "expr": "octez_validator_chain_head_cycle{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current cycle", + "refId": "A" + }, + { + "expr": "octez_store_caboose_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current caboose", + "refId": "B" + }, + { + "expr": "octez_store_checkpoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current checkpoint", + "refId": "C" + }, + { + "expr": "octez_store_savepoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current savepoint", + "refId": "D" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Chain levels", + "transform": "timeseries_to_rows", + "type": "table" + }, + { + "columns": [ ], + "datasource": "Prometheus", + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 15 + }, + "id": 10, + "links": [ ], + "styles": [ + { + "alias": "Time", + "pattern": "Time", + "type": "hidden" + } + ], + "targets": [ + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Incomming connections", + "refId": "A" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "B" + }, + { + "expr": "octez_p2p_connections_private{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Private points", + "refId": "C" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Connections", + "transform": "timeseries_to_rows", + "type": "table" + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 4, + "x": 0, + "y": 20 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 4, + "y": 3 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"0\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consensus", + "refId": "A" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"1\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Vote", + "refId": "B" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"2\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anonymous", + "refId": "C" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"3\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Manager", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Head level": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 6, + "x": 10, + "y": 11 + }, + "id": 13, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head level", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head level history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Validation time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 6, + "x": 4, + "y": 11 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validation time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block validation time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Gas consumed": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 10 + }, + "id": 15, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_consumed_gas{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Gas consumed", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Gas consumed history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Round": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 19 + }, + "id": 16, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Round", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Round history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Written block size": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 4, + "y": 20 + }, + "id": 17, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_written_block_size{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Written block size", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Last written block size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Alternate heads count": "yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 10, + "y": 20 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_alternate_heads_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alternate heads count", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Alternate heads count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 16, + "y": 0 + }, + "id": 19, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_store_invalid_blocks{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Invalid blocks mean", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 18, + "y": 0 + }, + "id": 20, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_branch_switch_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Branch switch count", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 20, + "y": 5 + }, + "id": 21, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_store_last_merge_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Merge time", + "refId": "A" + } + ], + "title": "Store merge time", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 22, + "y": 0 + }, + "id": 22, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Max round", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 16, + "y": 5 + }, + "id": 23, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "blocks validation mean", + "refId": "A" + } + ], + "title": "Blocks validation mean", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 18, + "y": 5 + }, + "id": 24, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Chain validator request completion", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": 0 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 20, + "y": 5 + }, + "id": 25, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_peer_system_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + }, + { + "expr": "octez_validator_peer_too_short_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "B" + }, + { + "expr": "octez_validator_peer_unavailable_protocol{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "C" + }, + { + "expr": "octez_validator_peer_unknown_ancestor{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "D" + }, + { + "expr": "octez_validator_peer_unknown_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "E" + } + ], + "title": "Peer validators errors", + "transformations": [ + { + "id": "calculateField", + "options": { + "mode": "reduceRow", + "reducer": { + "reducer": "sum" + }, + "replaceFields": true + } + } + ], + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 22, + "y": 5 + }, + "id": 26, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Incoming connections mean", + "transparent": false, + "type": "stat" + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_version,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez compact dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/octez-full.json b/tezt/lib_cloud/grafana/dashboards/octez-full.json new file mode 100644 index 0000000000000000000000000000000000000000..99793dc50bfc1631abf2fbff9015b9a786ef5e18 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/octez-full.json @@ -0,0 +1,6287 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Node stats", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bootstrapping" + }, + "1": { + "color": "green", + "index": 0, + "text": "Bootstrapped" + }, + "null": { + "color": "yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 3, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_is_bootstrapped{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bootstrap status", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Unsync" + }, + "1": { + "color": "green", + "index": 1, + "text": "Sync" + }, + "2": { + "color": "red", + "index": 2, + "text": "Stuck" + }, + "null": { + "color": "yellow", + "index": 3, + "text": "Unknow" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 4, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_synchronisation_status{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sync status", + "transparent": false, + "type": "stat" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ chain_name }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Chain name", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 8, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 11, + "y": 1 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_version{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Reflects the uptime of the monitoring of the job, not the uptime of the process.", + "format": "dtdhms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time()-(process_start_time_seconds{app=\"$node_instance\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "node uptime", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 7 + }, + "id": 9, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current head level", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current head level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 10 + }, + "id": 10, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ p2p_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "P2p version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 10 + }, + "id": 11, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ distributed_db_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Distributed db version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "value" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 13 + }, + "id": 12, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_savepoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current savepoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current savepoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 13 + }, + "id": 13, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_checkpoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current checkpoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current checkpoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 16 + }, + "id": 14, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_caboose_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current caboose", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current caboose level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 16 + }, + "id": 15, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_cycle{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current cycle", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current cycle", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 19 + }, + "id": 16, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trusted points", + "refId": "A" + } + ], + "thresholds": "", + "title": "Trusted points", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 19 + }, + "id": 17, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_connections_private{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private Connections", + "refId": "A" + } + ], + "thresholds": "", + "title": "Private Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "aliasColors": { + "Head level": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 4, + "y": 4 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head level", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head level history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Validation time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 14 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validation time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block validation time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Loki", + "gridPos": { + "h": 21, + "w": 10, + "x": 14, + "y": 0 + }, + "id": 20, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node logs", + "type": "logs" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 22 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"0\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consensus", + "refId": "A" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"1\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Vote", + "refId": "B" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"2\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anonymous", + "refId": "C" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"3\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Manager", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Invalid blocks": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 22 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_invalid_blocks{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Invalid blocks", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Invalid blocks history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Gas consumed": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 30 + }, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_consumed_gas{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Gas consumed", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Gas consumed history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Round": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 30 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Round", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Round history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Merge time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 38 + }, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_merge_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Merge time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Store merge time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Written block size": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 38 + }, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_written_block_size{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Written block size", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Last written block size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 47 + }, + "id": 27, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Node Hardware stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Cpu load": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 28, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(netdata_app_cpu_utilization_percentage_average{app_group=\"octez\",app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cpu load", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Cpu actitvity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Memory usage": "light-green", + "Swap usage": "light-orange" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_app_mem_usage_MiB_average{app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory usage", + "refId": "A" + }, + { + "expr": "netdata_app_swap_usage_MiB_average{app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Swap usage", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "mbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "mbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": null, + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "decbytes" + } + }, + "gridPos": { + "h": 8, + "w": 2, + "x": 0, + "y": 55 + }, + "id": 30, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "node_filesystem_free_bytes{mountpoint=\"/\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Available bytes on disk.", + "refId": "A" + } + ], + "title": "Disk free space", + "transparent": false, + "type": "stat" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 11, + "x": 2, + "y": 55 + }, + "id": 31, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_disk_space_GiB_average{chart=\"disk_space._\",dimension=\"used\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{dimension}}", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Storage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "reads": "light-green", + "writes": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 11, + "x": 13, + "y": 55 + }, + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_app_disk_logical_io_KiB_persec_average{app_group=\"octez\",dimension=\"reads\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "reads", + "refId": "A" + }, + { + "expr": "netdata_app_disk_logical_io_KiB_persec_average{app_group=\"octez\",dimension=\"writes\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "writes", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "IOs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "kbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "kbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": null, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + { + "alias": "/.*received/", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(node_network_receive_bytes_total[5m]) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bytes received", + "refId": "A" + }, + { + "expr": "irate(node_network_transmit_bytes_total[5m]) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Bytes transmitted", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Network traffic", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "All fds": "light-green", + "Files": "light-blue", + "Pipes": "light-orange", + "Sockets": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 34, + "legend": { + "alignAsTable": "true", + "avg": "true", + "current": "true", + "max": "true", + "min": "true", + "rightSide": "true", + "show": "true", + "sideWidth": null, + "total": false, + "values": "true" + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(netdata_app_fds_open_fds_average{dimension=\"sockets\",app_group=\"octez\",app=\"$node_instance\"}) + sum(netdata_app_fds_open_fds_average{dimension=\"files\",app_group=\"octez\",app=\"$node_instance\"}) + sum(netdata_app_fds_open_fds_average{dimension=\"pipes\",app_group=\"octez\",app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "All fds", + "refId": "A" + }, + { + "expr": "netdata_app_fds_open_fds_average{dimension=\"sockets\",app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Sockets", + "refId": "B" + }, + { + "expr": "netdata_app_fds_open_fds_average{dimension=\"files\",app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Files", + "refId": "C" + }, + { + "expr": "netdata_app_fds_open_fds_average{dimension=\"pipes\",app_group=\"octez\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Pipes", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "File descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 72 + }, + "id": 35, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "P2P stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Branch delayed": "light-red", + "Branch refused": "light-orange", + "Outdated": "blue", + "Refused": "red", + "Unprocessed": "light-yellow", + "Validated": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 72 + }, + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_mempool_pending_validated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validated", + "refId": "A" + }, + { + "expr": "octez_mempool_pending_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Refused", + "refId": "B" + }, + { + "expr": "octez_mempool_pending_outdated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outdated", + "refId": "C" + }, + { + "expr": "octez_mempool_pending_branch_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch refused", + "refId": "D" + }, + { + "expr": "octez_mempool_pending_branch_delayed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch delayed", + "refId": "E" + }, + { + "expr": "octez_mempool_pending_unprocessed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unprocessed", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Mempool status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 72 + }, + "id": 37, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected peers": "light-yellow", + "Running peers": "light-green", + "Waiting to reconnect": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 38, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_peers_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected peers", + "refId": "A" + }, + { + "expr": "octez_p2p_peers_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running peers", + "refId": "B" + }, + { + "expr": "octez_p2p_points_greylisted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Waiting to reconnect", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P peers connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected points": "light-red", + "Running points": "light-green", + "Trusted points": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 39, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_points_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected points", + "refId": "A" + }, + { + "expr": "octez_p2p_points_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running points", + "refId": "B" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P points connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current head broadcasted": "light-green", + "current head received": "light-blue", + "current head sent": "light-green", + "get current head broadcasted": "green", + "get current head received": "blue", + "get current head sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 88 + }, + "id": 40, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head broadcasted", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head sent", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head received", + "refId": "D" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head sent", + "refId": "E" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head broadcasted", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current head messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current branch received": "light-blue", + "current branch sent": "light-green", + "get current branch received": "blue", + "get current branch sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 88 + }, + "id": 41, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block header received": "light-blue", + "block header sent": "light-green", + "get block headers received": "blue", + "get block headers sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 88 + }, + "id": 42, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB block headers messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get predecessor header received": "blue", + "get predecessor header sent": "green", + "predecessor header received": "light-blue", + "predecessor header sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 88 + }, + "id": 43, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB predecessor header messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get operations received": "blue", + "get operations sent": "green", + "operation received": "light-blue", + "operation sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 96 + }, + "id": 44, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get ops for blocks received": "blue", + "get ops for blocks sent": "green", + "ops for block received": "light-blue", + "ops for block sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 96 + }, + "id": 45, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations for blocks messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocols received": "blue", + "get protocols sent": "green", + "protocol received": "light-blue", + "protocol sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 96 + }, + "id": 46, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocols messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocol branch received": "blue", + "get protocol branch sent": "green", + "protocol branch received": "light-blue", + "protocol branch sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 96 + }, + "id": 47, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocol branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "checkpoint received": "light-blue", + "checkpoint sent": "light-green", + "get checkpoint received": "blue", + "get checkpoint sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 96 + }, + "id": 48, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB checkpoint messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "deactivate received": "light-blue", + "deactivate sent": "light-green", + "get deactivate received": "blue", + "get deactivate sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 96 + }, + "id": 49, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB deactivate messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 105 + }, + "id": 50, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Workers stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 51, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator request count", + "refId": "A" + }, + { + "expr": "octez_validator_block_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator completion count", + "refId": "B" + }, + { + "expr": "octez_validator_block_worker_request_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator error count", + "refId": "C" + }, + { + "expr": "octez_validator_chain_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator request count", + "refId": "D" + }, + { + "expr": "octez_validator_chain_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator completion count", + "refId": "E" + }, + { + "expr": "octez_validator_chain_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator error count", + "refId": "F" + }, + { + "expr": "octez_mempool_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator request count", + "refId": "G" + }, + { + "expr": "octez_mempool_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator completion count", + "refId": "H" + }, + { + "expr": "octez_mempool_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator error count", + "refId": "I" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validation workers requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 105 + }, + "id": 52, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers table", + "refId": "A" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers scheduler", + "refId": "B" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations table", + "refId": "C" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations scheduler", + "refId": "D" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation table", + "refId": "E" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation scheduler", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB Workers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 121 + }, + "id": 53, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Treatments", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 121 + }, + "id": 54, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Completion", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "connections": "light-green", + "fetching canceled knew unknown head": "light-orange", + "fetching canceled new branch": "green", + "fetching canceled new known valid head": "light-yellow", + "invalid blocks": "light-red", + "invalid locator": "light-orange", + "new branch completed": "light-blue", + "new head completed": "blue", + "on no request count": "white", + "system error": "orange", + "too short locator": "brown", + "unavailable protocol": "light-red", + "unknown ancestor": "yellow", + "unknown error": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 129 + }, + "id": 55, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_peer_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "connections", + "refId": "A" + }, + { + "expr": "octez_validator_peer_invalid_block{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid blocks", + "refId": "B" + }, + { + "expr": "octez_validator_peer_invalid_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid locator", + "refId": "C" + }, + { + "expr": "octez_validator_peer_new_branch_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new branch completed", + "refId": "D" + }, + { + "expr": "octez_validator_peer_new_head_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new head completed", + "refId": "E" + }, + { + "expr": "octez_validator_peer_on_no_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "on no request count", + "refId": "F" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_branch{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new branch", + "refId": "G" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_known_valid_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new known valid head", + "refId": "H" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_unknown_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled knew unknown head", + "refId": "I" + }, + { + "expr": "octez_validator_peer_system_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "system error", + "refId": "J" + }, + { + "expr": "octez_validator_peer_too_short_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "too short locator", + "refId": "K" + }, + { + "expr": "octez_validator_peer_unavailable_protocol{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unavailable protocol", + "refId": "L" + }, + { + "expr": "octez_validator_peer_unknown_ancestor{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown ancestor", + "refId": "M" + }, + { + "expr": "octez_validator_peer_unknown_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown error", + "refId": "N" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Peer validators", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 16, + "w": 8, + "x": 0, + "y": 130 + }, + "id": 56, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "RPC", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 130 + }, + "id": 57, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC calls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 138 + }, + "id": 58, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/version.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/config.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/network.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/workers.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/injection.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/protocols.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/private.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 130 + }, + "id": 59, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total calls", + "refId": "A" + } + ], + "title": "Total of RPC calls", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 130 + }, + "id": 60, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(octez_rpc_calls_count{endpoint=~\".*\",app=\"$node_instance\"}[1h]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "calls rate", + "refId": "A" + } + ], + "title": "RPC calls rate per hour", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 138 + }, + "id": 61, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average duration", + "refId": "A" + } + ], + "title": "Average of calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 138 + }, + "id": 62, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"} / octez_rpc_calls_count{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max average duration", + "refId": "A" + } + ], + "title": "Max of average calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 138 + }, + "id": 63, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max total duration", + "refId": "A" + } + ], + "title": "Max of total calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 138 + }, + "id": 64, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total duration", + "refId": "A" + } + ], + "title": "Total of RPC calls duration", + "transparent": false, + "type": "stat" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 147 + }, + "id": 65, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Miscellaneous", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Forced major collections": "light-blue", + "Heap compactions": "light-red", + "Major collections": "light-yellow", + "Minor collections": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 147 + }, + "id": 66, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_minor_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Minor collections", + "refId": "A" + }, + { + "expr": "ocaml_gc_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major collections", + "refId": "B" + }, + { + "expr": "ocaml_gc_forced_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Forced major collections", + "refId": "C" + }, + { + "expr": "ocaml_gc_compactions{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Heap compactions", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC maintenance operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Major heap": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 147 + }, + "id": 67, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_heap_words{app=\"$node_instance\"}* 8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major heap", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC major word sizes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 156 + }, + "id": 68, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Delegates Hardware stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Cpu load accuser": "light-yellow", + "Cpu load baker": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 156 + }, + "id": 69, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_apps_cpu_percentage_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cpu load accuser", + "refId": "A" + }, + { + "expr": "netdata_apps_cpu_percentage_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Cpu load baker", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Cpu actitvity", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "percent", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Memory usage accuser": "dark-yellow", + "Memory usage baker": "dark-red", + "Swap usage accuser": "light-yellow", + "Swap usage baker": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 156 + }, + "id": 70, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_apps_mem_MiB_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory usage accuser", + "refId": "A" + }, + { + "expr": "netdata_apps_swap_MiB_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Swap usage accuser", + "refId": "B" + }, + { + "expr": "netdata_apps_mem_MiB_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Memory usage baker", + "refId": "C" + }, + { + "expr": "netdata_apps_swap_MiB_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Swap usage baker", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "mbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "mbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "reads accuser": "dark-yellow", + "reads baker": "dark-red", + "writes accuser": "light-yellow", + "writes baker": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 156 + }, + "id": 71, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "netdata_apps_lreads_KiB_persec_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "reads accuser", + "refId": "A" + }, + { + "expr": "netdata_apps_lwrites_KiB_persec_average{dimension=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "writes accuser", + "refId": "B" + }, + { + "expr": "netdata_apps_lreads_KiB_persec_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "reads baker", + "refId": "C" + }, + { + "expr": "netdata_apps_lweads_KiB_persec_average{dimension=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "writes baker", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "IOs", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "kbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "kbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 165 + }, + "id": 72, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Logs", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 165 + }, + "id": 73, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 165 + }, + "id": 74, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Baker logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Baker logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 165 + }, + "id": 75, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Accuser logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Accuser logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 175 + }, + "id": 76, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"varlogs\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "System logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System logs", + "type": "logs" + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez", + "hardware", + "logs" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_version,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez full dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/octez-with-logs.json b/tezt/lib_cloud/grafana/dashboards/octez-with-logs.json new file mode 100644 index 0000000000000000000000000000000000000000..45684421a8b19c0630f33d2b887e250a7a21268d --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/octez-with-logs.json @@ -0,0 +1,5309 @@ +{ + "__inputs": [ ], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Node stats", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 1, + "text": "Bootstrapping" + }, + "1": { + "color": "green", + "index": 0, + "text": "Bootstrapped" + }, + "null": { + "color": "yellow", + "index": 2, + "text": "Unknown" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 3, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_is_bootstrapped{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Bootstrap status", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ + { + "id": 0, + "options": { + "0": { + "color": "red", + "index": 0, + "text": "Unsync" + }, + "1": { + "color": "green", + "index": 1, + "text": "Sync" + }, + "2": { + "color": "red", + "index": 2, + "text": "Stuck" + }, + "null": { + "color": "yellow", + "index": 3, + "text": "Unknow" + } + }, + "type": "value" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 4, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "octez_validator_chain_synchronisation_status{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Sync status", + "transparent": false, + "type": "stat" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ chain_name }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Chain name", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 8, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 11, + "y": 1 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_version{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node release commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "description": "Reflects the uptime of the monitoring of the job, not the uptime of the process.", + "format": "dtdhms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 4 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time()-(process_start_time_seconds{app=\"$node_instance\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "node uptime", + "refId": "A" + } + ], + "thresholds": "", + "title": "Node uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 7 + }, + "id": 9, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current head level", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current head level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 10 + }, + "id": 10, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ p2p_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "P2p version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "name" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 10 + }, + "id": 11, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_version{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ distributed_db_version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Distributed db version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "value" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 13 + }, + "id": 12, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_savepoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current savepoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current savepoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 13 + }, + "id": 13, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_checkpoint_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current checkpoint", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current checkpoint level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 16 + }, + "id": 14, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_store_caboose_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "current caboose", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current caboose level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 16 + }, + "id": 15, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_validator_chain_head_cycle{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "Current cycle", + "refId": "A" + } + ], + "thresholds": "", + "title": "Current cycle", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 19 + }, + "id": 16, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "trusted points", + "refId": "A" + } + ], + "thresholds": "", + "title": "Trusted points", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 19 + }, + "id": 17, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_p2p_connections_private{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private Connections", + "refId": "A" + } + ], + "thresholds": "", + "title": "Private Connections", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "max" + }, + { + "aliasColors": { + "Head level": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 10, + "w": 10, + "x": 4, + "y": 4 + }, + "id": 18, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_level{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Head level", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head level history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Validation time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 4, + "y": 14 + }, + "id": 19, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validation time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Block validation time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Loki", + "gridPos": { + "h": 21, + "w": 10, + "x": 14, + "y": 0 + }, + "id": 20, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node logs", + "type": "logs" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 22 + }, + "id": 21, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"0\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Consensus", + "refId": "A" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"1\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Vote", + "refId": "B" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"2\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Anonymous", + "refId": "C" + }, + { + "expr": "octez_validator_block_operations_per_pass{pass_id=\"3\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Manager", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Invalid blocks": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 22 + }, + "id": 22, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_invalid_blocks{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Invalid blocks", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Invalid blocks history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Gas consumed": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 30 + }, + "id": 23, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_consumed_gas{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Gas consumed", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Gas consumed history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "sci", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Round": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 30 + }, + "id": 24, + "legend": { + "alignAsTable": true, + "avg": false, + "current": false, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_head_round{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Round", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Round history", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Merge time": "light-blue" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 14, + "x": 0, + "y": 38 + }, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_merge_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Merge time", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Store merge time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Written block size": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 10, + "x": 14, + "y": 38 + }, + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_store_last_written_block_size{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Written block size", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Last written block size", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 47 + }, + "id": 27, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "P2P stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Branch delayed": "light-red", + "Branch refused": "light-orange", + "Outdated": "blue", + "Refused": "red", + "Unprocessed": "light-yellow", + "Validated": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "decimals": 0, + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 28, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": true, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_mempool_pending_validated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Validated", + "refId": "A" + }, + { + "expr": "octez_mempool_pending_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Refused", + "refId": "B" + }, + { + "expr": "octez_mempool_pending_outdated{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outdated", + "refId": "C" + }, + { + "expr": "octez_mempool_pending_branch_refused{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch refused", + "refId": "D" + }, + { + "expr": "octez_mempool_pending_branch_delayed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Branch delayed", + "refId": "E" + }, + { + "expr": "octez_mempool_pending_unprocessed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Unprocessed", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Mempool status", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "decimals": 0, + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Incoming connections": "light-green", + "Outgoing connections": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 29, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_connections_outgoing{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Outgoing connections", + "refId": "A" + }, + { + "expr": "octez_p2p_connections_incoming{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Incoming connections", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P total connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected peers": "light-yellow", + "Running peers": "light-green", + "Waiting to reconnect": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_peers_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected peers", + "refId": "A" + }, + { + "expr": "octez_p2p_peers_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running peers", + "refId": "B" + }, + { + "expr": "octez_p2p_points_greylisted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Waiting to reconnect", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P peers connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Disconnected points": "light-red", + "Running points": "light-green", + "Trusted points": "light-yellow" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 55 + }, + "id": 31, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_p2p_points_disconnected{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Disconnected points", + "refId": "A" + }, + { + "expr": "octez_p2p_points_running{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Running points", + "refId": "B" + }, + { + "expr": "octez_p2p_points_trusted{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Trusted points", + "refId": "C" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "P2P points connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current head broadcasted": "light-green", + "current head received": "light-blue", + "current head sent": "light-green", + "get current head broadcasted": "green", + "get current head received": "blue", + "get current head sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 63 + }, + "id": 32, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head broadcasted", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head sent", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_get_current_head_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current head received", + "refId": "D" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head sent", + "refId": "E" + }, + { + "expr": "octez_distributed_db_message_current_head_messages{action=\"broadcasted\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current head broadcasted", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current head messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "current branch received": "light-blue", + "current branch sent": "light-green", + "get current branch received": "blue", + "get current branch sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 63 + }, + "id": 33, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_current_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get current branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_current_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "current branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB current branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block header received": "light-blue", + "block header sent": "light-green", + "get block headers received": "blue", + "get block headers sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 63 + }, + "id": 34, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_block_headers_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get block headers received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_block_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB block headers messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get predecessor header received": "blue", + "get predecessor header sent": "green", + "predecessor header received": "light-blue", + "predecessor header sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 63 + }, + "id": 35, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_predecessor_header_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get predecessor header received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_predecessor_header_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "predecessor header sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB predecessor header messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get operations received": "blue", + "get operations sent": "green", + "operation received": "light-blue", + "operation sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 71 + }, + "id": 36, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get operations received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operation_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get ops for blocks received": "blue", + "get ops for blocks sent": "green", + "ops for block received": "light-blue", + "ops for block sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 6, + "x": 6, + "y": 71 + }, + "id": 37, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_operations_for_blocks_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get ops for blocks received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_operations_for_block_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "ops for block sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB operations for blocks messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocols received": "blue", + "get protocols sent": "green", + "protocol received": "light-blue", + "protocol sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 71 + }, + "id": 38, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocols_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocols received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocols messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "get protocol branch received": "blue", + "get protocol branch sent": "green", + "protocol branch received": "light-blue", + "protocol branch sent": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 71 + }, + "id": 39, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_protocol_branch_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get protocol branch received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_protocol_branch_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "protocol branch sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB protocol branch messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "checkpoint received": "light-blue", + "checkpoint sent": "light-green", + "get checkpoint received": "blue", + "get checkpoint sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 71 + }, + "id": 40, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_checkpoint_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get checkpoint received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_checkpoint_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "checkpoint sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB checkpoint messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "deactivate received": "light-blue", + "deactivate sent": "light-green", + "get deactivate received": "blue", + "get deactivate sent": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 71 + }, + "id": 41, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate sent", + "refId": "A" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate received", + "refId": "B" + }, + { + "expr": "octez_distributed_db_message_get_deactivate_messages{action=\"received\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "get deactivate received", + "refId": "C" + }, + { + "expr": "octez_distributed_db_message_deactivate_messages{action=\"sent\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deactivate sent", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB deactivate messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 80 + }, + "id": 42, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Workers stats", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 80 + }, + "id": 43, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_block_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator request count", + "refId": "A" + }, + { + "expr": "octez_validator_block_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator completion count", + "refId": "B" + }, + { + "expr": "octez_validator_block_worker_request_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Block validator error count", + "refId": "C" + }, + { + "expr": "octez_validator_chain_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator request count", + "refId": "D" + }, + { + "expr": "octez_validator_chain_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator completion count", + "refId": "E" + }, + { + "expr": "octez_validator_chain_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chain validator error count", + "refId": "F" + }, + { + "expr": "octez_mempool_worker_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator request count", + "refId": "G" + }, + { + "expr": "octez_mempool_worker_completion_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator completion count", + "refId": "H" + }, + { + "expr": "octez_mempool_worker_error_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Prevalidator validator error count", + "refId": "I" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validation workers requests", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 80 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers table", + "refId": "A" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"block_header\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "headers scheduler", + "refId": "B" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations table", + "refId": "C" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operations\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operations scheduler", + "refId": "D" + }, + { + "expr": "octez_distributed_db_requester_table_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation table", + "refId": "E" + }, + { + "expr": "octez_distributed_db_requester_scheduler_length{requester_kind=\"operation\",app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "operation scheduler", + "refId": "F" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "DDB Workers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 96 + }, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_push_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Treatments", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "block validator": "light-red", + "chain validator": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 96 + }, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_chain_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_chain_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "chain validator", + "refId": "A" + }, + { + "expr": "octez_validator_block_last_finished_request_completion_timestamp{app=\"$node_instance\"} - octez_validator_block_last_finished_request_treatment_timestamp{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "block validator", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Validators Requests Completion", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "connections": "light-green", + "fetching canceled knew unknown head": "light-orange", + "fetching canceled new branch": "green", + "fetching canceled new known valid head": "light-yellow", + "invalid blocks": "light-red", + "invalid locator": "light-orange", + "new branch completed": "light-blue", + "new head completed": "blue", + "on no request count": "white", + "system error": "orange", + "too short locator": "brown", + "unavailable protocol": "light-red", + "unknown ancestor": "yellow", + "unknown error": "red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 104 + }, + "id": 47, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_validator_peer_connections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "connections", + "refId": "A" + }, + { + "expr": "octez_validator_peer_invalid_block{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid blocks", + "refId": "B" + }, + { + "expr": "octez_validator_peer_invalid_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "invalid locator", + "refId": "C" + }, + { + "expr": "octez_validator_peer_new_branch_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new branch completed", + "refId": "D" + }, + { + "expr": "octez_validator_peer_new_head_completed{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "new head completed", + "refId": "E" + }, + { + "expr": "octez_validator_peer_on_no_request_count{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "on no request count", + "refId": "F" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_branch{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new branch", + "refId": "G" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_known_valid_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled new known valid head", + "refId": "H" + }, + { + "expr": "octez_validator_peer_operations_fetching_canceled_new_unknown_head{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "fetching canceled knew unknown head", + "refId": "I" + }, + { + "expr": "octez_validator_peer_system_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "system error", + "refId": "J" + }, + { + "expr": "octez_validator_peer_too_short_locator{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "too short locator", + "refId": "K" + }, + { + "expr": "octez_validator_peer_unavailable_protocol{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unavailable protocol", + "refId": "L" + }, + { + "expr": "octez_validator_peer_unknown_ancestor{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown ancestor", + "refId": "M" + }, + { + "expr": "octez_validator_peer_unknown_error{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "unknown error", + "refId": "N" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Peer validators", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 16, + "w": 8, + "x": 0, + "y": 105 + }, + "id": 48, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "RPC", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 105 + }, + "id": 49, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC calls", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Blocks": "light-blue", + "Chains": "light-red", + "Config": "light-yellow", + "Injection": "green", + "Misc": "grey", + "Network": "blue", + "Private": "purple", + "Protocols": "blue", + "Versions": "light-green", + "Workers": "light-red" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 113 + }, + "id": 50, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Chains", + "refId": "A" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/chains//blocks/.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Blocks", + "refId": "B" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/version.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/version.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Versions", + "refId": "C" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/config.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/config.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Config", + "refId": "D" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/network.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/network.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Network", + "refId": "E" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/workers.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/workers.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Workers", + "refId": "F" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/injection.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/injection.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Injection", + "refId": "G" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/protocols.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/protocols.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Protocols", + "refId": "H" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/private.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/private.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Private", + "refId": "I" + }, + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'/(fetch_protocol|stats|monitor)*.',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Misc", + "refId": "J" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "RPC durations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 12, + "y": 105 + }, + "id": 51, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total calls", + "refId": "A" + } + ], + "title": "Total of RPC calls", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "light-yellow", + "value": "Base" + } + ] + }, + "unit": "none" + } + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 18, + "y": 105 + }, + "id": 52, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(rate(octez_rpc_calls_count{endpoint=~\".*\",app=\"$node_instance\"}[1h]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "calls rate", + "refId": "A" + } + ], + "title": "RPC calls rate per hour", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 12, + "y": 113 + }, + "id": 53, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})/sum(octez_rpc_calls_count{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "average duration", + "refId": "A" + } + ], + "title": "Average of calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 15, + "y": 113 + }, + "id": 54, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"} / octez_rpc_calls_count{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max average duration", + "refId": "A" + } + ], + "title": "Max of average calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 18, + "y": 113 + }, + "id": 55, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "max" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "max(octez_rpc_calls_sum{app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "max total duration", + "refId": "A" + } + ], + "title": "Max of total calls durations", + "transparent": false, + "type": "stat" + }, + { + "datasource": "Prometheus", + "fieldConfig": { + "defaults": { + "links": [ ], + "mappings": [ ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": "Base" + } + ] + }, + "unit": "s" + } + }, + "gridPos": { + "h": 8, + "w": 3, + "x": 21, + "y": 113 + }, + "id": 56, + "links": [ ], + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "7", + "targets": [ + { + "expr": "sum(octez_rpc_calls_sum{endpoint=~'.*',app=\"$node_instance\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total duration", + "refId": "A" + } + ], + "title": "Total of RPC calls duration", + "transparent": false, + "type": "stat" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 122 + }, + "id": 57, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Miscellaneous", + "titleSize": "h6", + "type": "row" + }, + { + "aliasColors": { + "Forced major collections": "light-blue", + "Heap compactions": "light-red", + "Major collections": "light-yellow", + "Minor collections": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 122 + }, + "id": 58, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_minor_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Minor collections", + "refId": "A" + }, + { + "expr": "ocaml_gc_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major collections", + "refId": "B" + }, + { + "expr": "ocaml_gc_forced_major_collections{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Forced major collections", + "refId": "C" + }, + { + "expr": "ocaml_gc_compactions{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Heap compactions", + "refId": "D" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC maintenance operations", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Major heap": "light-green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 122 + }, + "id": 59, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ocaml_gc_heap_words{app=\"$node_instance\"}* 8", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Major heap", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "GC major word sizes", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 0, + "w": 8, + "x": 0, + "y": 131 + }, + "id": 60, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Logs", + "titleSize": "h6", + "type": "row" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 131 + }, + "id": 61, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Node logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Node logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 131 + }, + "id": 62, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-baker\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Baker logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Baker logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 131 + }, + "id": 63, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"octez-accuser\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Accuser logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Accuser logs", + "type": "logs" + }, + { + "datasource": "Loki", + "gridPos": { + "h": 14, + "w": 12, + "x": 0, + "y": 141 + }, + "id": 64, + "options": { + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "span": 12, + "targets": [ + { + "expr": "{job=\"varlogs\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "System logs", + "refId": "A" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "System logs", + "type": "logs" + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez", + "logs" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_version,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez with logs dashboard", + "version": 0 +} diff --git a/tezt/lib_cloud/grafana/dashboards/rollup.json b/tezt/lib_cloud/grafana/dashboards/rollup.json new file mode 100644 index 0000000000000000000000000000000000000000..dfbbcb6e01ab680890740d8b1704a21df22464c5 --- /dev/null +++ b/tezt/lib_cloud/grafana/dashboards/rollup.json @@ -0,0 +1,1213 @@ +{ + "__inputs": [], + "__requires": [ ], + "annotations": { + "list": [ ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ ], + "panels": [ + { + "collapse": false, + "collapsed": false, + "id": 2, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Version info", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 3, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ version }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 4, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_date }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Git Commit Date", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(label_replace(octez_sc_rollup_node_node_info{app=\"$node_instance\"},\"commit_hash_short\" ,\"$1\",\"commit_hash\",\"^(.{8}).*$\"))", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ commit_hash_short }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Git Commit", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 12, + "w": 17, + "x": 0, + "y": 5 + }, + "id": 6, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Generic information", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 5 + }, + "id": 7, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ genesis_level }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Genesis Level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 5 + }, + "id": 8, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ pvm_kind }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Pvm Kind", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 9, + "x": 8, + "y": 5 + }, + "id": 9, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ genesis_hash }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Genesis Hash", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 2, + "w": 9, + "x": 8, + "y": 7 + }, + "id": 10, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_node_info{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "{{ rollup_address }}", + "refId": "A" + } + ], + "thresholds": "", + "title": "Rollup Address", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapse": false, + "collapsed": false, + "gridPos": { + "h": 12, + "w": 17, + "x": 0, + "y": 17 + }, + "id": 11, + "panels": [ ], + "repeat": "", + "repeatIteration": null, + "repeatRowId": null, + "showTitle": true, + "title": "Inbox", + "titleSize": "h6", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 17 + }, + "id": 12, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "octez_sc_rollup_node_head_inbox_level{app=\"$node_instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Head Inbox Level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 17 + }, + "id": 13, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "rate(octez_sc_rollup_node_head_inbox_internal_messages_number{app=\"$node_instance\"}[15m])+ rate(octez_sc_rollup_node_head_inbox_external_messages_number{app=\"$node_instance\"}[15m])", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Genesis Level", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 17 + }, + "id": 14, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "rate(octez_sc_rollup_node_head_inbox_internal_messages_number{app=\"$node_instance\"}[15m])", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "Internal Messages Rate", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "Prometheus", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 17 + }, + "id": 15, + "interval": null, + "links": [ ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "rate(octez_sc_rollup_node_head_inbox_internal_messages_number{app=\"$node_instance\"}[15m])", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "", + "title": "External Messages Rate", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": { + "External Messages": "yellow", + "Internal messages": "green" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 20 + }, + "id": 16, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_sc_rollup_node_head_inbox_internal_messages_number{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Internal messages", + "refId": "A" + }, + { + "expr": "octez_sc_rollup_node_head_inbox_external_messages_number{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "External Messages", + "refId": "B" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Inbox Messages", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "Prometheus", + "fill": 1, + "fillGradient": 0, + "gridPos": { + "h": 9, + "w": 16, + "x": 0, + "y": 29 + }, + "id": 17, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": null, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [ ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "octez_sc_rollup_node_head_inbox_process_time{app=\"$node_instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time (in s)", + "refId": "A" + } + ], + "thresholds": [ ], + "timeFrom": null, + "timeShift": null, + "title": "Head process time", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "", + "rows": [ ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "tezos", + "octez", + "rollups" + ], + "templating": { + "list": [ + { + "allValue": null, + "current": { }, + "datasource": "Prometheus", + "hide": 0, + "includeAll": false, + "label": "Node instance", + "multi": false, + "name": "node_instance", + "options": [ ], + "query": "label_values(octez_sc_rollup_node_node_info,app)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [ ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Octez Rollup Node", + "version": 0 +} diff --git a/tezt/lib_cloud/jobs.ml b/tezt/lib_cloud/jobs.ml index cf09e9aea2a642f42057e153fefd911ebb7c70da..c24b605b671b99c9c9751ecb2a7634678293f260 100644 --- a/tezt/lib_cloud/jobs.ml +++ b/tezt/lib_cloud/jobs.ml @@ -10,133 +10,90 @@ module C = Cli open Tezt module Cli = C -let docker_build ~push () = - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "TEZT_CLOUD environment variable found with value: %s" tezt_cloud ; - let ssh_public_key = Lazy.force Env.ssh_public_key in - Log.info "Checking the existence of ssh public key '%s'..." ssh_public_key ; - let* ssh_public_key = - let ssh_public_key_file = Lazy.force Env.ssh_public_key in - let* () = - if not (Sys.file_exists ssh_public_key_file) then Ssh.generate_key () - else Lwt.return_unit - in - Process.run_and_read_stdout ~name:"cat" "cat" [ssh_public_key_file] - in - let dockerfile = Lazy.force Env.dockerfile in - Log.info - "Checking the existence of the docker file %s.Dockerfile..." - dockerfile ; - if not (Sys.file_exists dockerfile) then - Test.fail - "Could not find the dockerfile %s. See README for more information why \ - this dockerfile is necessary." - dockerfile ; - Log.info "Checking the existence of the zcash parameters..." ; - if not (Sys.file_exists Path.zcash_params) then - Test.fail - "Could not find the zcash parameters in %s. See README for more \ - information why such files are necessary." - Path.zcash_params ; - Log.info "Initializing terraform docker state..." ; - let* () = Terraform.Docker_registry.init () in - Log.info "Fetching or creating the remote docker registry..." ; - let* hostname = Terraform.Docker_registry.get_hostname () in - let* docker_registry = Terraform.Docker_registry.get_docker_registry () in - Log.info "Authenticate docker hostname..." ; - let* () = Gcloud.auth_configure_docker ~hostname in - let args = - [ - ("SSH_PUBLIC_KEY", ssh_public_key); - ("ZCASH_PARAMS_PATH", Path.zcash_params); - ("DAL_TRUSTED_SETUP_PATH", Path.dal_trusted_setup); - ("BINARIES_DESTINATION_PATH", Path.default_binaries_path ()); - ] - in - Log.info "Building image from %s.Dockerfile..." tezt_cloud ; - let*! () = Docker.build ~dockerfile ~args () in - Log.info "Tagging the image..." ; - let*! () = Docker.tag docker_registry in - Log.info "Push the image..." ; - let* () = - if push then - let*! () = Docker.push docker_registry in - Lwt.return_unit - else Lwt.return_unit - in - unit - -let register_docker_push ~tags = - Test.register - ~__FILE__ - ~title:"Push the dockerfile to the registry" - ~tags:("docker" :: "push" :: tags) - @@ fun () -> docker_build ~push:true () +let docker_build = + let cache = Hashtbl.create 11 in + fun ?(docker_image = Env.Gcp {alias = Env.dockerfile_alias}) ~push () -> + if Hashtbl.mem cache docker_image then ( + Log.info "Docker image is already built. Nothing to do" ; + Lwt.return_unit) + else + let ssh_public_key_filename = Env.ssh_public_key_filename () in + Hashtbl.replace cache docker_image () ; + Log.info + "Checking the existence of ssh public key '%s'..." + ssh_public_key_filename ; + let* ssh_public_key = + let* () = + if not (Sys.file_exists ssh_public_key_filename) then ( + Log.info "SSH public key not found, creating it..." ; + Ssh.generate_key ()) + else Lwt.return_unit + in + Process.run_and_read_stdout ~name:"cat" "cat" [ssh_public_key_filename] + in + let alias = + match docker_image with + | Gcp {alias} -> alias + | Octez_latest_release -> "octez" + in + let dockerfile = Path.dockerfile ~alias in + Log.info "Checking the existence of the docker file %s..." dockerfile ; + if not (Sys.file_exists dockerfile) then + Test.fail + "Could not find the dockerfile %s. See README for more information \ + why this dockerfile is necessary." + dockerfile ; + Log.info "Checking the existence of the zcash parameters..." ; + if (not (Sys.file_exists Path.zcash_params)) && alias = Env.tezt_cloud + then + Test.fail + "Could not find the zcash parameters in %s. See README for more \ + information why such files are necessary." + Path.zcash_params ; + if + (not (Sys.file_exists Path.dal_trusted_setup)) && alias = Env.tezt_cloud + then + Test.fail + "Could not find the dal_trusted_setup in %s. See README for more \ + information why such files are necessary." + Path.dal_trusted_setup ; + let args = + match docker_image with + | Gcp _ -> + [ + ("SSH_PUBLIC_KEY", ssh_public_key); + ("ZCASH_PARAMS_PATH", Path.zcash_params); + ("DAL_TRUSTED_SETUP_PATH", Path.dal_trusted_setup); + ("BINARIES_DESTINATION_PATH", Path.default_binaries_path ()); + ] + | Octez_latest_release -> [("SSH_PUBLIC_KEY", ssh_public_key)] + in + Log.info "Building image from %s..." Env.tezt_cloud ; + let* () = Docker.build ~alias ~args () |> Process.check in + let* () = + if push then ( + Log.info "Compute the registry uri..." ; + let* registry_uri = Env.registry_uri () in + Log.info "Tagging the image..." ; + let* () = Docker.tag ~alias ~registry_uri () |> Process.check in + Log.info "Push the image..." ; + let* () = Docker.push ~alias ~registry_uri () |> Process.check in + Lwt.return_unit) + else Lwt.return_unit + in + unit let deploy_docker_registry () = - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "Tezt_Cloud found with value: %s" tezt_cloud ; + Log.info "Tezt_Cloud found with value: %s" Env.tezt_cloud ; let* () = Terraform.Docker_registry.init () in - Terraform.Docker_registry.deploy () - -let register_deploy_docker_registry ~tags = - Test.register - ~__FILE__ - ~title:"Deploy docker registry" - ~tags:("docker" :: "registry" :: "deploy" :: tags) - deploy_docker_registry - -let deploy_terraform_state_bucket ~tags = - Test.register - ~__FILE__ - ~title:"Deploy terraform state bucket" - ~tags:("terraform" :: "state" :: "bucket" :: "deploy" :: tags) - @@ fun () -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "Tezt_Cloud found with value: %s" tezt_cloud ; - let* () = Terraform.State_bucket.init () in - Terraform.State_bucket.deploy () - -let destroy_vms ~tags = - Test.register - ~__FILE__ - ~title:"Destroy terraform VMs" - ~tags:("terraform" :: "destroy" :: "vms" :: tags) - @@ fun () -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "Tezt_Cloud found with value: %s" tezt_cloud ; - let* () = Terraform.VM.Workspace.select "default" in - let* workspaces = Terraform.VM.Workspace.get () in - let* () = Terraform.VM.destroy workspaces in - Terraform.VM.Workspace.destroy () + let tezt_cloud = Env.tezt_cloud in + let* project_id = Gcloud.project_id () in + Terraform.Docker_registry.deploy ~tezt_cloud ~project_id -let prometheus_import ~tags = - Test.register - ~__FILE__ - ~title:"Import a snapshot into a prometheus container" - ~tags:("prometheus" :: "import" :: tags) - @@ fun () -> - match Cli.prometheus_snapshot with - | None -> - Test.fail "Use --prometheus-snapshot to provide the snapshot filename" - | Some filename -> - let* prometheus = - Prometheus.run_with_snapshot - ~snapshot:(Cli.prometheus_snapshot_directory // filename) - ~port:9090 - in - Log.info "You can find the prometheus instance at http://localhost:9090" ; - Log.info - "Use Ctrl+C to end the scenario and kill the prometheus instance." ; - Prometheus.shutdown prometheus - -let clean_up_vms ~tags = - Test.register - ~__FILE__ - ~title:"Clean ups VMs manually" - ~tags:("clean" :: "up" :: tags) - @@ fun () -> - let* workspaces = Terraform.VM.Workspace.get () in +let clean_up_vms () = + let tezt_cloud = Env.tezt_cloud in + let* workspaces = Terraform.VM.Workspace.list ~tezt_cloud in + let ssh_private_key_filename = Env.ssh_private_key_filename () in workspaces |> Lwt_list.iter_s (fun workspace -> let* () = Terraform.VM.Workspace.select workspace in @@ -153,19 +110,25 @@ let clean_up_vms ~tags = let* () = names |> Lwt_list.iter_p (fun vm_name -> - let*! output = + let* output = Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["ps"; "--format"; "{{.Names}}"] + |> Process.check_and_read_stdout in let images_name = String.split_on_char '\n' output |> List.filter (fun str -> str <> "") in let main_image, other_images = - List.partition (fun str -> str <> "netdata") images_name + List.partition + (fun str -> + str <> "netdata" && str <> "grafana" + && str <> "prometheus") + images_name in if List.length main_image <> 1 then Test.fail @@ -174,56 +137,38 @@ let clean_up_vms ~tags = in this list" (String.concat ";" images_name) ; let main_image = List.hd main_image in - let*! _ = + let* _ = Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["stop"; main_image] + |> Process.check in - let*! _ = + let* _ = Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["start"; main_image] + |> Process.check in let* () = other_images |> Lwt_list.iter_p (fun image -> - let*! _ = + let* _ = Gcloud.compute_ssh ~zone ~vm_name + ~ssh_private_key_filename "docker" ["kill"; image] - in - let*! _ = - Gcloud.compute_ssh - ~zone - ~vm_name - "docker" - ["rm"; image] + |> Process.check in Lwt.return_unit) in Lwt.return_unit) in unit) - -let list_vms ~tags = - Test.register ~__FILE__ ~title:"List VMs" ~tags:("list" :: "vms" :: tags) - @@ fun () -> - let tezt_cloud = Lazy.force Env.tezt_cloud in - Log.info "TEZT_CLOUD environment variable found with value: %s" tezt_cloud ; - let* _ = Gcloud.list_vms ~prefix:tezt_cloud in - Lwt.return_unit - -let register ~tags = - register_docker_push ~tags ; - register_deploy_docker_registry ~tags ; - deploy_terraform_state_bucket ~tags ; - destroy_vms ~tags ; - prometheus_import ~tags ; - clean_up_vms ~tags ; - list_vms ~tags diff --git a/tezt/lib_cloud/jobs.mli b/tezt/lib_cloud/jobs.mli index 45780b8476a377ace60ae74c34375ecdb683d861..8592b4ef52d35509f95ac564f655b362d73da2de 100644 --- a/tezt/lib_cloud/jobs.mli +++ b/tezt/lib_cloud/jobs.mli @@ -9,9 +9,8 @@ val deploy_docker_registry : unit -> unit Lwt.t (* A job for building and pushing docker images on the registry. *) -val docker_build : push:bool -> unit -> unit Lwt.t +val docker_build : + ?docker_image:Env.docker_image -> push:bool -> unit -> unit Lwt.t -(** [register ~tags] register a set of jobs that can be used for setting - requirements related to cloud scenarios. Some tags can be given for all the - registered jobs. *) -val register : tags:string list -> unit +(* The docker containers are killed and restarted from scratch. *) +val clean_up_vms : unit -> unit Lwt.t diff --git a/tezt/lib_cloud/monitoring.ml b/tezt/lib_cloud/monitoring.ml index 165e18a02c21ce57dd4f6696756d7f64f329fe69..dd77374c31de439108a02afdf4896b5827daeaa5 100644 --- a/tezt/lib_cloud/monitoring.ml +++ b/tezt/lib_cloud/monitoring.ml @@ -5,9 +5,11 @@ (* *) (*****************************************************************************) -let run ~run_command = +let run ?cmd_wrapper () = let* is_already_running = - let*? process = run_command "docker" ["ps"; "--format"; "{{.Names}}"] in + let process = + Env.run_command ?cmd_wrapper "docker" ["ps"; "--format"; "{{.Names}}"] + in let* status = Process.wait process in match status with | Unix.WEXITED 0 -> @@ -26,18 +28,17 @@ let run ~run_command = (* Those arguments are the ones given by netdata on their online documentation. We could decide to put it in the eDSL module of docker. *) let args = - "run -d --name=netdata --pid=host --network=host -v \ + "run -d --rm --name=netdata --pid=host --network=host -v \ netdataconfig:/etc/netdata -v netdatalib:/var/lib/netdata -v \ netdatacache:/var/cache/netdata -v /etc/passwd:/host/etc/passwd:ro \ -v /etc/group:/host/etc/group:ro -v \ /etc/localtime:/etc/localtime:ro -v /proc:/host/proc:ro -v \ /sys:/host/sys:ro -v /etc/os-release:/host/etc/os-release:ro -v \ /var/log:/host/var/log:ro -v \ - /var/run/docker.sock:/var/run/docker.sock:ro --restart \ - unless-stopped --cap-add SYS_PTRACE --cap-add SYS_ADMIN \ - --security-opt apparmor=unconfined netdata/netdata" - |> String.split_on_char ' ' + /var/run/docker.sock:/var/run/docker.sock:ro --cap-add SYS_PTRACE \ + --cap-add SYS_ADMIN --security-opt apparmor=unconfined \ + netdata/netdata" |> String.split_on_char ' ' |> List.filter (fun s -> s <> "") in - let*! _ = run_command cmd args in + let* _ = Env.run_command ?cmd_wrapper cmd args |> Process.check in Lwt.return_unit diff --git a/tezt/lib_cloud/monitoring.mli b/tezt/lib_cloud/monitoring.mli index c7ff51b953953ba8e4984b89a68b67dd3f589897..53594f012b66dfef59b29881286f94ac4084f547 100644 --- a/tezt/lib_cloud/monitoring.mli +++ b/tezt/lib_cloud/monitoring.mli @@ -5,8 +5,5 @@ (* *) (*****************************************************************************) -(** [run ~run_command] runs netdata on the VM. [run_command] is a callback to - run a command on the VM. *) -val run : - run_command:(string -> string list -> (Process.t, string) Runnable.t) -> - unit Lwt.t +(** [run ?cmd_wrapper ()] runs netdata on the VM. *) +val run : ?cmd_wrapper:Gcloud.cmd_wrapper -> unit -> unit Lwt.t diff --git a/tezt/lib_cloud/path.ml b/tezt/lib_cloud/path.ml index b059333a0f3bce646fd7031addb65c2acdf989b5..a289484aa395615b54f6978b723ed5d9377248bc 100644 --- a/tezt/lib_cloud/path.ml +++ b/tezt/lib_cloud/path.ml @@ -19,10 +19,19 @@ let terraform_state_bucket = terraform // "state-bucket" let terraform_vm = terraform // "vm" -let docker = project // "dockerfiles" +let dockerfile ~alias = + project // "dockerfiles" // Format.asprintf "%s.Dockerfile" alias let zcash_params = "_opam" // "share" // "zcash-params" let dal_trusted_setup = "_opam" // "share" // "dal-trusted-setup" let default_binaries_path () = Filename.get_temp_dir_name () // "tezt-runners" + +let self = Sys.argv.(0) + +let proxy_deployement ~tezt_cloud = + Filename.get_temp_dir_name () + // Format.asprintf "%s-tezt-cloud-deployement" tezt_cloud + +let grafana_dashboards = project // "grafana" // "dashboards" diff --git a/tezt/lib_cloud/path.mli b/tezt/lib_cloud/path.mli index d6800b8eeef17e0f873a378ace2f9d1fd331806e..7d39ac1f11699b33869658e8ac1e5ff319e32fda 100644 --- a/tezt/lib_cloud/path.mli +++ b/tezt/lib_cloud/path.mli @@ -18,7 +18,7 @@ val terraform_state_bucket : string val terraform_vm : string (** Path to the directory containing dockerfiles for each user. *) -val docker : string +val dockerfile : alias:string -> string (** Path to the zcash_params directory that can be used for the dockerfiles. *) @@ -30,3 +30,12 @@ val dal_trusted_setup : string (** Default path where images can be found on the docker container. *) val default_binaries_path : unit -> string + +(** Path to the current binary. *) +val self : string + +(** Path where to find the encoding of tezt cloud deployement. *) +val proxy_deployement : tezt_cloud:string -> string + +(** Path where are stored grafana dashboards. *) +val grafana_dashboards : string diff --git a/tezt/lib_cloud/prometheus.ml b/tezt/lib_cloud/prometheus.ml index e298496596410e4e4fff522496541404e85281f2..50fcfb03855ea6d040f13bf63cdae71e38bbe93d 100644 --- a/tezt/lib_cloud/prometheus.ml +++ b/tezt/lib_cloud/prometheus.ml @@ -13,6 +13,8 @@ type t = { configuration_file : string; mutable sources : source list; scrape_interval : int; + snapshot_filename : string option; + port : int; } let netdata_source_of_agents agents = @@ -64,14 +66,21 @@ let tezt_source = { job_name = "tezt_metrics"; metric_path = "/metrics.txt"; - targets = [{address = "localhost"; port = 8080; app_name = "tezt"}]; + targets = + [ + { + address = "localhost"; + port = (if Env.mode = `Orchestrator then 80 else 8080); + app_name = "tezt"; + }; + ]; } let config ~scrape_interval sources = let sources = List.map str_of_source sources |> String.concat "" in prefix ~scrape_interval () ^ sources -let write_configuration_file {scrape_interval; configuration_file; sources} = +let write_configuration_file {scrape_interval; configuration_file; sources; _} = let config = config ~scrape_interval sources in with_open_out configuration_file (fun oc -> Stdlib.seek_out oc 0 ; @@ -89,13 +98,25 @@ let add_source t ?(metric_path = "/metrics") ~job_name targets = write_configuration_file t ; reload t -let start ?(scrape_interval = 5) agents = +let start agents = let sources = - if Cli.monitoring then [tezt_source; netdata_source_of_agents agents] + if Env.monitoring then [tezt_source; netdata_source_of_agents agents] else [tezt_source] in - let configuration_file = Temp.file "prometheus.yml" in - let t = {configuration_file; sources; scrape_interval} in + let* () = + Process.run "mkdir" ["-p"; Filename.get_temp_dir_name () // "prometheus"] + in + (* We do not use the Temp.dir so that the base directory is predictable and + can be mounted by the proxy VM if [--proxy] is used. *) + let configuration_file = + Filename.get_temp_dir_name () // "prometheus" // "prometheus.yml" + in + let snapshot_filename = Env.prometheus_snapshot_filename in + let port = Env.prometheus_port in + let scrape_interval = Env.prometheus_scrape_interval in + let t = + {configuration_file; sources; scrape_interval; snapshot_filename; port} + in write_configuration_file t ; let process = Process.spawn @@ -108,8 +129,8 @@ let start ?(scrape_interval = 5) agents = "prometheus"; "--network"; "host"; - "-p"; - "9090:9090"; + (* We use the host mode so that in [localhost], prometheus can see the + metrics endpoint run by other docker containers. *) "-v"; Format.asprintf "%s:/etc/prometheus/prometheus.yml" configuration_file; "prom/prometheus"; @@ -118,6 +139,8 @@ let start ?(scrape_interval = 5) agents = (* To export a snapshot. *) "--web.enable-lifecycle"; (* To reload the configuration while prometheus is running.*) + Format.asprintf "--web.listen-address=:%d" port; + (* Specify the port on which the prometheus instance runs. *) ] in let* status = Process.wait process in @@ -133,43 +156,25 @@ let start ?(scrape_interval = 5) agents = (* Fail if something unexpected happens. *) Process.check process in - let rec wait_for_ready () = - let process = - Process.spawn - "curl" - [ - "-s"; - "-o"; - "/dev/null"; - "-w"; - "%{http_code}"; - "http://localhost:9090/-/ready"; - ] - in - let* status = Process.wait process in - match status with - | Unix.WEXITED 0 -> - let* status = Process.check_and_read_stdout process in - if String.trim status = "200" then Lwt.return_unit - else ( - Log.info - "Prometheus container is not ready, let's wait 2 seconds and check \ - again..." ; - let* () = Lwt_unix.sleep 2. in - wait_for_ready ()) - | _ -> - Log.info - "Prometheus container is not ready, let's wait 2 seconds and check \ - again..." ; - let* () = Lwt_unix.sleep 2. in - wait_for_ready () + let is_ready output = String.trim output = "200" in + let run () = + Process.spawn + "curl" + [ + "-s"; + "-o"; + "/dev/null"; + "-w"; + "%{http_code}"; + "http://localhost:9090/-/ready"; + ] in - let* () = wait_for_ready () in + let* _ = Env.wait_process ~is_ready ~run () in Lwt.return t let snapshots_path = "/prometheus" // "data" // "snapshots" -let export_snapshot {configuration_file = _; _} = +let export_snapshot {snapshot_filename; _} = Log.info "Exporting snapshot..." ; let* stdout = Process.run_and_read_stdout @@ -179,27 +184,41 @@ let export_snapshot {configuration_file = _; _} = let json = JSON.parse ~origin:"Prometheus.export" stdout in let snapshot_name = JSON.(json |-> "data" |-> "name" |> as_string) in let destination = - match Cli.prometheus_snapshot with - | None -> Cli.prometheus_snapshot_directory - | Some file -> Cli.prometheus_snapshot_directory // file + Option.value + ~default:(Filename.get_temp_dir_name () // snapshot_name) + snapshot_filename in - let*! () = + let* () = Docker.cp "prometheus" ~kind:`To_host ~source:(snapshots_path // snapshot_name) ~destination + |> Process.check in Log.info "You can find the prometheus snapshot at %s" destination ; Lwt.return_unit let shutdown {configuration_file = _; _} = - let*! () = Docker.kill "prometheus" in + let* () = Docker.kill "prometheus" |> Process.check in Lwt.return_unit -let run_with_snapshot ~snapshot ~port = +let run_with_snapshot () = (* No need for a configuration file here. *) let configuration_file = "" in + let port = Env.prometheus_port in + let snapshot_filename = + match Env.prometheus_snapshot_filename with + | None -> + Test.fail + "You must specify the snapshot filename via \ + --prometheus-snapshot-filename" + | Some file -> file + in + Log.info + "You can find the prometheus instance at http://localhost:%d" + Env.prometheus_port ; + Log.info "Use Ctrl+C to end the scenario and kill the prometheus instance." ; let* () = Process.run "docker" @@ -207,7 +226,7 @@ let run_with_snapshot ~snapshot ~port = "run"; "-uroot"; "-v"; - Format.asprintf "%s:/prometheus" snapshot; + Format.asprintf "%s:/prometheus" snapshot_filename; "-p"; Format.asprintf "%d:9090" port; "prom/prometheus"; @@ -215,4 +234,11 @@ let run_with_snapshot ~snapshot ~port = "--storage.tsdb.path=/prometheus"; ] in - Lwt.return {configuration_file; sources = []; scrape_interval = 0} + Lwt.return + { + configuration_file; + sources = []; + scrape_interval = 0; + snapshot_filename = Some snapshot_filename; + port; + } diff --git a/tezt/lib_cloud/prometheus.mli b/tezt/lib_cloud/prometheus.mli index e12fe452debee18810c08bb131976834c6ac931e..047b9beb82abb7da6f078edf83440756a9e44e62 100644 --- a/tezt/lib_cloud/prometheus.mli +++ b/tezt/lib_cloud/prometheus.mli @@ -9,9 +9,8 @@ type t type target = {address : string; port : int; app_name : string} -(** [start ?scrape_interval agents] starts the prometheus server. - [scrape_interval] is set to [5] seconds by default. *) -val start : ?scrape_interval:int -> Agent.t List.t -> t Lwt.t +(** [start agents] starts the prometheus server. *) +val start : Agent.t List.t -> t Lwt.t (** [shutdown prometheus] shutdowns the prometheus server. *) val shutdown : t -> unit Lwt.t @@ -19,9 +18,10 @@ val shutdown : t -> unit Lwt.t (** [export_snapshot prometheus] exports a prometheus snapshot. *) val export_snapshot : t -> unit Lwt.t -(** [run_wtih_snapshot ~snapshot ~port] allows to run the prometheus with the - given snapshot on port [port]. *) -val run_with_snapshot : snapshot:string -> port:int -> t Lwt.t +(** [run_wtih_snapshot ()] allows to run the prometheus with the given + snapshot. The snapshot must have been exported by running a tezt-cloud + scenario beforehand. *) +val run_with_snapshot : unit -> t Lwt.t (** [reload prometheus] reloads the prometheus configuration. This can be used to take into account a change such a different agent name or a new source. *) diff --git a/tezt/lib_cloud/proxy.ml b/tezt/lib_cloud/proxy.ml new file mode 100644 index 0000000000000000000000000000000000000000..90b0900a990fa649b449d3308f2533b613ec18d8 --- /dev/null +++ b/tezt/lib_cloud/proxy.ml @@ -0,0 +1,102 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +let find_agent agents = + agents + |> List.find_opt (fun agent -> + let proxy_agent_prefix = Format.asprintf "%s-proxy" Env.tezt_cloud in + String.starts_with ~prefix:proxy_agent_prefix (Agent.name agent)) + +let get_agent agents = find_agent agents |> Option.get + +let copy_files proxy_agent ~scenario_files ~proxy_deployement = + (* This file is necessary to get the agents configurations. *) + let* _ = + Agent.copy + proxy_agent + ~source:proxy_deployement + ~destination:proxy_deployement + in + (* Copying the ssh key is necessary for the proxy agent to connect with the + other VMs. *) + let ssh_public_key_filename = Env.ssh_public_key_filename () in + let ssh_private_key_filename = Env.ssh_private_key_filename () in + let* _ = + Agent.copy + proxy_agent + ~source:ssh_public_key_filename + ~destination: + ("/root" // ".ssh" // Filename.basename ssh_public_key_filename) + in + let* _ = + Agent.copy + proxy_agent + ~source:ssh_private_key_filename + ~destination: + ("/root" // ".ssh" // Filename.basename ssh_private_key_filename) + in + (* If the Proxy agent uses grafana, it needs some dashboards. We copy them to + the proxy VM and then import them. + + Another way to do it, would be to push them from the host machine to the + grafana instance directly. This is doable and slightly better in theory. + *) + let* _ = + if Env.grafana then + let* grafana_dashboards = Grafana.dashboards_filepaths () in + grafana_dashboards + |> List.map (fun dashboard -> + let* _ = + Agent.copy + proxy_agent + ~source:dashboard + ~destination: + (Filename.get_temp_dir_name () + // "grafana" // "dashboards" + // Filename.basename dashboard) + in + Lwt.return_unit) + |> Lwt.join + else Lwt.return_unit + in + (* The proxy agent requires the daemons to be in the current working + directory. We add symbolink links to all the known binaries. + + This requires the docker image to contain all the binaries used by the + proxy agent. + *) + let* output = + Process.spawn + ~runner:(Agent.runner proxy_agent) + "ls" + [Path.default_binaries_path ()] + |> Process.check_and_read_stdout + in + let files = String.trim output |> String.split_on_char '\n' in + let* () = + files + |> List.map (fun file -> + Process.spawn + ~runner:(Agent.runner proxy_agent) + "ln" + ["-s"; Path.default_binaries_path () // file; file; "-f"] + |> Process.check) + |> Lwt.join + in + (* The scenario itself may need some files that exist on the host machine but + are not present by default on the proxy machine. Since it may be difficult to + know ahead which files are needed, instead we require the scenario to provide + them manually. *) + let* () = + List.map + (fun file -> + let* _ = Agent.copy proxy_agent ~source:file ~destination:file in + Lwt.return_unit) + scenario_files + |> Lwt.join + in + unit diff --git a/tezt/lib_cloud/proxy.mli b/tezt/lib_cloud/proxy.mli new file mode 100644 index 0000000000000000000000000000000000000000..ce6a3ba220e000c91dc7701bc3e0c7e770203db7 --- /dev/null +++ b/tezt/lib_cloud/proxy.mli @@ -0,0 +1,19 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(** [get_agent agents] returns the proxy agent. It raises [Not_found] if the + proxy agent was not found. This function should be safe to call when + [Env.mode] is [`Orchestrator] or [`Host]. *) +val get_agent : Agent.t list -> Agent.t + +(** [copy_files agent] copies all the necessary files for the proxy orchestrator + to run correctly. *) +val copy_files : + Agent.t -> + scenario_files:string list -> + proxy_deployement:string -> + unit Lwt.t diff --git a/tezt/lib_cloud/ssh.ml b/tezt/lib_cloud/ssh.ml index 68ff9cced734594745a9108f3e41da1556fc0317..86d7f429106d82593a43eeef146092c4015a70f9 100644 --- a/tezt/lib_cloud/ssh.ml +++ b/tezt/lib_cloud/ssh.ml @@ -6,7 +6,7 @@ (*****************************************************************************) let generate_key () = - let path = Format.asprintf "%s" (Lazy.force Env.ssh_private_key) in + let path = Format.asprintf "%s" (Env.ssh_private_key_filename ()) in (* -C is for comment; -N is for the passphrase (no passphrase here) *) diff --git a/tezt/lib_cloud/ssh.mli b/tezt/lib_cloud/ssh.mli index 8bb8a58b1b6670786d87bb23f4c87422570913a9..c17679024736e062f1fa872094118e0041f5322c 100644 --- a/tezt/lib_cloud/ssh.mli +++ b/tezt/lib_cloud/ssh.mli @@ -5,6 +5,6 @@ (* *) (*****************************************************************************) -(* [generate_key ()] generates an ssh key based on the - [Env.tezt_cloud] variable environment. *) +(** [generate_key ()] generates an ssh key based on the [Env.tezt_cloud] + variable environment. *) val generate_key : unit -> unit Lwt.t diff --git a/tezt/lib_cloud/terraform.ml b/tezt/lib_cloud/terraform.ml index 81bc98662b2332002baf351b56f591f9b2e1a8d8..c2e859d75d1aa376661674e95e21879264705f0e 100644 --- a/tezt/lib_cloud/terraform.ml +++ b/tezt/lib_cloud/terraform.ml @@ -21,9 +21,7 @@ module Docker_registry = struct "terraform" (chdir Path.terraform_docker_registry @ ["init"]) - let deploy () = - let* project_id = Gcloud.project_id () in - let tezt_cloud = Lazy.force Env.tezt_cloud in + let deploy ~tezt_cloud ~project_id = Process.run ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) ~name @@ -37,8 +35,7 @@ module Docker_registry = struct Format.asprintf "project_id=%s" project_id; ]) - let get_docker_registry () = - let tezt_cloud = Lazy.force Env.tezt_cloud in + let get_docker_registry ~tezt_cloud = let* output = Process.run_and_read_stdout ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) @@ -47,14 +44,13 @@ module Docker_registry = struct "terraform" (chdir Path.terraform_docker_registry @ ["output"; "-json"]) in - let json = JSON.parse ~origin:"docker_registry" output in + let json = JSON.parse ~origin:"get_docker_registry" output in let registry_name = JSON.(json |-> "docker_registry" |-> "value" |> as_string) in Lwt.return registry_name - let get_hostname () = - let tezt_cloud = Lazy.force Env.tezt_cloud in + let get_hostname ~tezt_cloud = let* output = Process.run_and_read_stdout ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) @@ -63,9 +59,22 @@ module Docker_registry = struct "terraform" (chdir Path.terraform_docker_registry @ ["output"; "-json"]) in - let json = JSON.parse ~origin:"docker_registry_hostname" output in + let json = JSON.parse ~origin:"get_hostname" output in let registry_name = JSON.(json |-> "hostname" |-> "value" |> as_string) in Lwt.return registry_name + + let get_zone ~tezt_cloud = + let* output = + Process.run_and_read_stdout + ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) + ~name + ~color + "terraform" + (chdir Path.terraform_docker_registry @ ["output"; "-json"]) + in + let json = JSON.parse ~origin:"get_zone" output in + let registry_name = JSON.(json |-> "zone" |-> "value" |> as_string) in + Lwt.return registry_name end module VM = struct @@ -81,8 +90,7 @@ module VM = struct (* Return all the workspaces associated with the current tezt cloud environment. *) - let get () = - let tezt_cloud = Lazy.force Env.tezt_cloud in + let list ~tezt_cloud = (* We select the default workspace to be sure we can parse correctly the output. *) let* () = select "default" in @@ -96,14 +104,18 @@ module VM = struct String.split_on_char '\n' output |> List.map String.trim |> List.filter (fun workspace -> - String.starts_with ~prefix:tezt_cloud workspace) + let prefix = Format.asprintf "%s-" tezt_cloud in + String.starts_with ~prefix workspace + (* The workspace named "tezt-cloud" is used for the docker + registry, it does not need to be listed. *) + && workspace <> tezt_cloud) |> Lwt.return - (* Create workspaces that will be used for the experiment. Delete the ones - that won't be used. *) - let init workspaces = - let* existing_workspaces = get () in + (* Create workspaces that will be used for the experiment. *) + let init ~tezt_cloud workspaces = + let* existing_workspaces = list ~tezt_cloud in let to_create = + (* Create only new workspaces. *) List.filter (fun workspace -> not @@ List.mem workspace existing_workspaces) workspaces @@ -124,8 +136,10 @@ module VM = struct let* () = select "default" in unit - let destroy () = - let* workspaces = get () in + let destroy ~tezt_cloud = + (* We ensure we are not using a workspace we want to delete. *) + let* () = select "default" in + let* workspaces = list ~tezt_cloud in workspaces |> List.map (fun workspace -> Process.run @@ -210,9 +224,7 @@ module VM = struct in Lwt.return machine_type - let destroy workspaces = - let* project_id = Gcloud.project_id () in - (* let docker_image = Env.custom_docker_image ~project_id in *) + let destroy workspaces ~project_id = workspaces |> Lwt_list.iter_s (fun workspace -> let* () = Workspace.select workspace in @@ -241,29 +253,3 @@ module VM = struct "terraform" (chdir Path.terraform_vm @ ["destroy"; "--auto-approve"] @ vars)) end - -module State_bucket = struct - let init () = - let* () = Docker_registry.init () in - Process.run - ~name - ~color - "terraform" - (chdir Path.terraform_state_bucket @ ["init"]) - - let deploy () = - let tezt_cloud = Lazy.force Env.tezt_cloud in - let* project_id = Gcloud.project_id () in - Process.run - ~env:(String_map.singleton "TF_WORKSPACE" tezt_cloud) - ~name - ~color - "terraform" - (chdir Path.terraform_state_bucket - @ [ - "apply"; - "--auto-approve"; - "--var"; - Format.asprintf "project_id=%s" project_id; - ]) -end diff --git a/tezt/lib_cloud/terraform.mli b/tezt/lib_cloud/terraform.mli index 6707c97933adf78d21c38eb2711d53f8d2151736..49026d7b3e1be96387b6883417e89a386d6e431a 100644 --- a/tezt/lib_cloud/terraform.mli +++ b/tezt/lib_cloud/terraform.mli @@ -8,22 +8,24 @@ module Docker_registry : sig val init : unit -> unit Lwt.t - val deploy : unit -> unit Lwt.t + val deploy : tezt_cloud:string -> project_id:string -> unit Lwt.t - val get_docker_registry : unit -> string Lwt.t + val get_docker_registry : tezt_cloud:string -> string Lwt.t - val get_hostname : unit -> string Lwt.t + val get_hostname : tezt_cloud:string -> string Lwt.t + + val get_zone : tezt_cloud:string -> string Lwt.t end module VM : sig module Workspace : sig - val init : string list -> unit Lwt.t + val init : tezt_cloud:string -> string list -> unit Lwt.t - val get : unit -> string list Lwt.t + val list : tezt_cloud:string -> string list Lwt.t val select : string -> unit Lwt.t - val destroy : unit -> unit Lwt.t + val destroy : tezt_cloud:string -> unit Lwt.t end val init : unit -> unit Lwt.t @@ -41,11 +43,5 @@ module VM : sig val zone : unit -> string Lwt.t - val destroy : string list -> unit Lwt.t -end - -module State_bucket : sig - val init : unit -> unit Lwt.t - - val deploy : unit -> unit Lwt.t + val destroy : string list -> project_id:string -> unit Lwt.t end diff --git a/tezt/lib_cloud/terraform/docker-registry/main.tf b/tezt/lib_cloud/terraform/docker-registry/main.tf index a01cd88040b7a0852975f1a2b097f0d194687a8d..cb976243caa7fa09b114455b2d134454c1b70c52 100644 --- a/tezt/lib_cloud/terraform/docker-registry/main.tf +++ b/tezt/lib_cloud/terraform/docker-registry/main.tf @@ -32,6 +32,12 @@ terraform { version = "~> 5.0" } } + + backend "gcs" { + # Generic bucket state that contains all the states for all the + # workspaces + bucket = "dal-tfstate" + } } resource "google_artifact_registry_repository" "docker-registry" { @@ -61,3 +67,8 @@ output "docker_registry" { value = local.registry description = "URL of the GCP docker registry for this region" } + +output "zone" { + value = var.zone + description = "Zone of the docker registry" +} diff --git a/tezt/lib_cloud/terraform/vm/main.tf b/tezt/lib_cloud/terraform/vm/main.tf index 4ff8b0fa328da9fc81fd8b30d8fc6dc39d3083ae..2a22acc8c5f806a34ab3a683c5079100df37b974 100644 --- a/tezt/lib_cloud/terraform/vm/main.tf +++ b/tezt/lib_cloud/terraform/vm/main.tf @@ -99,7 +99,63 @@ module "gce-container" { source = "terraform-google-modules/container-vm/google" version = "~> 3.0" - container = { image = "${var.docker_image}" } + container = { image = "${var.docker_image}" + + # Volume settings is only necessary for the proxy VM + volumeMounts = [ + { + # Using the proxy mode, this is necessary if the docker image runs another docker image + mountPath = "/var/run/docker.sock" + name = "docker-socket" + readOnly = false + }, + { + # Necessary to provide access from the image docker to the website + mountPath = "/tmp/website" + name = "website" + readOnly = false + }, + { + # Same for Prometheus + mountPath = "/tmp/prometheus" + name = "prometheus" + readOnly = false + }, + { + # Same for Grafana + mountPath = "/tmp/grafana" + name = "grafana" + readOnly = false + } + ] + } + + volumes = [ + { + name = "docker-socket" + hostPath = { + path = "/var/run/docker.sock" + } + }, + { + name = "website" + hostPath = { + path = "/tmp/website" + } + }, + { + name = "prometheus" + hostPath = { + path = "/tmp/prometheus" + } + }, + { + name = "grafana" + hostPath = { + path = "/tmp/grafana" + } + } + ] } # When running a VM, it must be associated with a Virtual Private @@ -161,6 +217,24 @@ resource "google_compute_firewall" "default" { ports = ["19999"] } + # Rule to enable static page web access + allow { + protocol = "tcp" + ports = ["80", "8080"] + } + + # Rule to enable prometheus access + allow { + protocol = "tcp" + ports = ["9090"] + } + + # Rule to enable grafana access + allow { + protocol = "tcp" + ports = ["3000"] + } + # Anybody can contact the machine on the open ports. source_ranges = ["0.0.0.0/0"] } @@ -178,7 +252,7 @@ resource "google_compute_instance_template" "default" { project = var.project_id - name = "instance-template" + name_prefix = "instance-template-${terraform.workspace}" service_account { # Google recommends custom service accounts that have cloud-platform scope and permissions granted via IAM Roles. diff --git a/tezt/lib_cloud/tezt_cloud.ml b/tezt/lib_cloud/tezt_cloud.ml index b2440bd430f72d66ea343cbb5ce257afa6c52396..1e0ae5ebd4f8571a6631f52039150327975a860f 100644 --- a/tezt/lib_cloud/tezt_cloud.ml +++ b/tezt/lib_cloud/tezt_cloud.ml @@ -5,6 +5,147 @@ (* *) (*****************************************************************************) +module Agent = Agent + +module Configuration = struct + include Env + include Configuration +end + module Cloud = Cloud -let register ~tags = Jobs.register ~tags +let register_docker_push ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Push the dockerfile to the GCP registry" + ~tags:("docker" :: "push" :: tags) + @@ fun _cloud -> Jobs.docker_build ~push:true () + +let register_docker_build ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Build the dockerfile" + ~tags:("docker" :: "build" :: tags) + @@ fun _cloud -> Jobs.docker_build ~push:false () + +let register_deploy_docker_registry ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Deploy docker registry" + ~tags:("docker" :: "registry" :: "deploy" :: tags) + @@ fun _cloud -> Jobs.deploy_docker_registry () + +let register_destroy_vms ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Destroy terraform VMs" + ~tags:("terraform" :: "destroy" :: "vms" :: tags) + @@ fun _cloud -> + let tezt_cloud = Env.tezt_cloud in + let* project_id = Gcloud.project_id () in + let* workspaces = Terraform.VM.Workspace.list ~tezt_cloud in + let* () = Terraform.VM.destroy workspaces ~project_id in + Terraform.VM.Workspace.destroy ~tezt_cloud + +let register_prometheus_import ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Import a snapshot into a prometheus container" + ~tags:("prometheus" :: "import" :: tags) + @@ fun _cloud -> + let* prometheus = Prometheus.run_with_snapshot () in + Prometheus.shutdown prometheus + +let register_clean_up_vms ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Clean ups VMs manually" + ~tags:("clean" :: "up" :: tags) + @@ fun _cloud -> Jobs.clean_up_vms () + +let register_list_vms ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"List VMs" + ~tags:("list" :: "vms" :: tags) + @@ fun _cloud -> + Log.info "TEZT_CLOUD environment variable found with value: %s" Env.tezt_cloud ; + let* _ = Gcloud.list_vms ~prefix:Env.tezt_cloud in + Lwt.return_unit + +let register_create_dns_zone ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Create a new DNS zone" + ~tags:("create" :: "dns" :: "zone" :: tags) + @@ fun _cloud -> + let domain = + match Env.dns_domain with + | None -> + Test.fail "You must specify the domain to use via --dns-domain option." + | Some domain -> domain + in + let* () = Gcloud.DNS.create_zone ~domain ~zone:"tezt-cloud" () in + let* _ = Gcloud.DNS.describe ~zone:"tezt-cloud" () in + unit + +let register_describe_dns_zone ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Describe a new DNS zone" + ~tags:("describe" :: "dns" :: "zone" :: tags) + @@ fun _cloud -> + let* _ = Gcloud.DNS.describe ~zone:"tezt-cloud" () in + unit + +let register_dns_add ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Register a new DNS entry" + ~tags:("dns" :: "add" :: tags) + @@ fun _cloud -> + let tezt_cloud = Env.tezt_cloud in + let ip = Cli.get_string_opt "ip" in + match ip with + | None -> Test.fail "You must provide an IP address via -a ip=" + | Some ip -> + let* () = Gcloud.DNS.add ~tezt_cloud ~zone:"tezt-cloud" ~ip in + unit + +let register_dns_remove ~tags = + Cloud.register + ?vms:None + ~__FILE__ + ~title:"Remove a DNS entry" + ~tags:("dns" :: "remove" :: tags) + @@ fun _cloud -> + let tezt_cloud = Env.tezt_cloud in + let ip = Cli.get_string_opt "ip" in + match ip with + | None -> Test.fail "You must provide an IP address via -a ip=" + | Some ip -> + let* () = Gcloud.DNS.remove ~tezt_cloud ~zone:"tezt-cloud" ~ip in + unit + +let register ~tags = + register_docker_push ~tags ; + register_docker_build ~tags ; + register_deploy_docker_registry ~tags ; + register_destroy_vms ~tags ; + register_prometheus_import ~tags ; + register_clean_up_vms ~tags ; + register_list_vms ~tags ; + register_create_dns_zone ~tags ; + register_describe_dns_zone ~tags ; + register_dns_add ~tags ; + register_dns_remove ~tags diff --git a/tezt/lib_cloud/tezt_cloud.mli b/tezt/lib_cloud/tezt_cloud.mli index 895f14cc92bb5be537e029a643bcb4bb9d2e1d3c..fbe6364cdb8014d23fc23a306b44465a88ebd42d 100644 --- a/tezt/lib_cloud/tezt_cloud.mli +++ b/tezt/lib_cloud/tezt_cloud.mli @@ -5,21 +5,23 @@ (* *) (*****************************************************************************) -module Cloud : sig +module Agent : sig type t - module Configuration : sig - type docker_image = - | Custom of {tezt_cloud : string} - | Image of {docker_image : string} + val configuration : Agent.t -> Configuration.t +end + +module Configuration : sig + type docker_image = Gcp of {alias : string} | Octez_latest_release - type t = private { - machine_type : string; - docker_image : docker_image; - max_run_duration : int option; - } + type t = private { + machine_type : string; + docker_image : docker_image; + max_run_duration : int option; + binaries_path : string; + } - (** [make ?max_run_duration ?machine_type ?docker_image ()] is a smart-constructor to make a VM + (** [make ?machine_type ()] is a smart-constructor to make a VM configuration. Default value for [max_run_duration] is [7200]. @@ -29,19 +31,23 @@ module Cloud : sig Default value for [docker_image] is [Custom {tezt_cloud}] where [tezt_cloud] is the value provided by the environement variable [$TEZT_CLOUD]. *) - val make : - ?max_run_duration:int -> - ?machine_type:string -> - ?docker_image:docker_image -> - unit -> - t - end + val make : + ?binaries_path:string -> + ?max_run_duration:int -> + ?machine_type:string -> + ?docker_image:docker_image -> + unit -> + t +end + +module Cloud : sig + type t (** A wrapper around [Test.register] that can be used to register new tests using VMs provided as a map indexed by name. Each VM is abstracted via the [Agent] module. *) val register : - ?docker_push:bool -> + ?proxy_files:string list -> ?vms:Configuration.t list -> __FILE__:string -> title:string -> @@ -72,8 +78,6 @@ module Cloud : sig points to scrap. Each point can have a name defined by [app_name]. *) val add_prometheus_source : t -> ?metric_path:string -> job_name:string -> target list -> unit Lwt.t - - val get_configuration : t -> Agent.t -> Configuration.t end (** [register ~tags] register a set of jobs that can be used for setting diff --git a/tezt/lib_cloud/web.ml b/tezt/lib_cloud/web.ml index 274a3f3d2edca32cca0eba7e1fb91f42a7e71cd1..b986b00eb8c42aea27b2a4f4baf74a294a8351c5 100644 --- a/tezt/lib_cloud/web.ml +++ b/tezt/lib_cloud/web.ml @@ -5,25 +5,98 @@ (* *) (*****************************************************************************) -type t = {process : Process.t; dir : string; deployement : Deployement.t} +type t = { + process : Process.t; + dir : string; + monitoring : bool; + prometheus : bool; +} -let configuration ~deployement ~agents = +let pp_docker_image fmt = function + | Env.Gcp {alias} -> Format.fprintf fmt "%s" alias + | Octez_latest_release -> Format.fprintf fmt "Octez latest release" + +let configuration ~agents = let str = agents |> List.map (fun agent -> - let configuration = - Deployement.get_configuration deployement agent - in + let configuration = Agent.configuration agent in Format.asprintf - "- %s: %s" + {| +## %s + +- **Machine type**: %s +- **Docker_image**: %a + + |} (Agent.name agent) - configuration.machine_type) + configuration.machine_type + pp_docker_image + configuration.docker_image) |> String.concat "\n" in Format.asprintf "# Configurations@.%s\n" str +let string_docker_command agent = + let point = Agent.point agent in + let ssh_id = (Agent.runner agent).Runner.ssh_id in + String.concat + " " + [ + "ssh"; + Format.asprintf "root@%s" (fst point); + "-p"; + string_of_int (snd point); + "-o"; + "StrictHostKeyChecking=no"; + "-i"; + ssh_id |> Option.get; + ] + +let string_vm_command agent = + match Agent.cmd_wrapper agent with + | None -> "# Just run the command, the VM is the host machine" + | Some cmd_wrapper -> + String.concat " " (cmd_wrapper.Gcloud.cmd :: cmd_wrapper.args) + +let debugging ~agents = + let str = + agents + |> List.map (fun agent -> + let host_run_command = + Printf.sprintf + {| +```bash +%s +``` + |} + (string_vm_command agent) + in + let docker_command = + Printf.sprintf {| +```bash +%s +``` +|} (string_docker_command agent) + in + Printf.sprintf + {| +## %s +Connect on the VM: +%s + +Connect on the Docker: +%s +|} + (Agent.name agent) + host_run_command + docker_command) + |> String.concat "\n" + in + Format.asprintf "# Debugging@.%s\n" str + let monitoring ~agents = - if Cli.monitoring then + if Env.monitoring then let str = agents |> List.map (fun agent -> @@ -40,13 +113,37 @@ let monitoring ~agents = else "# Monitoring\n Monitoring disabled. Use `--monitoring` to activate it.\n" -let prometheus () = - if Cli.prometheus then - "# Prometheus\n [Prometheus dashboard](http://localhost:9090)" - else "Prometheus disabled. Use `--prometheus` to activate it." +let prometheus ~agents = + let domain = + match Env.mode with + | `Orchestrator -> Proxy.get_agent agents |> Agent.point |> fst + | `Host | `Localhost | `Cloud -> "localhost" + in + if Env.prometheus then + Format.asprintf + "# Prometheus\n [Prometheus dashboard](http://%s:%d)\n" + domain + Env.prometheus_port + else "Prometheus disabled. Use `--prometheus` to activate it.\n" -let markdown_content ~deployement ~agents = - [configuration ~deployement ~agents; monitoring ~agents; prometheus ()] +let grafana ~agents = + let domain = + match Env.mode with + | `Orchestrator -> Proxy.get_agent agents |> Agent.point |> fst + | `Host | `Localhost | `Cloud -> "localhost" + in + if Env.grafana then + Format.asprintf "# Grafana\n [Grafana dashboard](http://%s:3000)\n" domain + else "Grafana disabled. Use `--grafana` to activate it.\n" + +let markdown_content ~agents = + [ + configuration ~agents; + grafana ~agents; + prometheus ~agents; + monitoring ~agents; + debugging ~agents; + ] |> String.concat "\n" let index dir = dir // "index.md" @@ -54,7 +151,7 @@ let index dir = dir // "index.md" let write t ~agents = (* The content is formatted in markdown and will be rendered in html via pandoc. *) - let content = markdown_content ~deployement:t.deployement ~agents in + let content = markdown_content ~agents in let dir = t.dir in let index = index dir in Base.with_open_out index (fun oc -> output_string oc content) ; @@ -72,9 +169,15 @@ let write t ~agents = "-s"; ] -let run ~port ~deployement = - let dir = Temp.dir "website" in +let run () = + let* () = + Process.run "mkdir" ["-p"; Filename.get_temp_dir_name () // "website"] + in + let dir = Filename.get_temp_dir_name () // "website" in let index = index dir in + let port = Env.website_port in + let prometheus = Env.prometheus in + let monitoring = Env.monitoring in let process = Process.spawn "python3" @@ -86,10 +189,10 @@ let run ~port ~deployement = Filename.dirname index; ] in - Lwt.return {process; dir; deployement} + Lwt.return {process; dir; monitoring; prometheus} -let start ~port ~deployement ~agents = - let* t = run ~port ~deployement in +let start ~agents = + let* t = run () in let* () = write t ~agents in Lwt.return t diff --git a/tezt/lib_cloud/web.mli b/tezt/lib_cloud/web.mli index a22637d05e2c6a287c0439f113378d2a41afffa8..7aaea7e4c6148977fe5845b5449e06762387d9e1 100644 --- a/tezt/lib_cloud/web.mli +++ b/tezt/lib_cloud/web.mli @@ -7,10 +7,9 @@ type t -(** [start ~port ~agents] starts a webpage with experimentations information +(** [start ~agents env] starts a webpage with experimentations information if [Cli.website] is [true]. *) -val start : - port:int -> deployement:Deployement.t -> agents:Agent.t List.t -> t Lwt.t +val start : agents:Agent.t List.t -> t Lwt.t (** [shutdown website] shutdowns the website. *) val shutdown : t -> unit Lwt.t diff --git a/tezt/lib_performance_regression/grafana.ml b/tezt/lib_performance_regression/grafana.ml index 0aad67d7b36997a7c367bd519bbb763bba83e2fc..f05f4d4608302ca4d93d4bb3de3cdcff3047bcef 100644 --- a/tezt/lib_performance_regression/grafana.ml +++ b/tezt/lib_performance_regression/grafana.ml @@ -246,12 +246,12 @@ let handle_http_error resp_status resp_body request = body req -let update_dashboard config dashboard = - if dashboard.uid =~! uid_rex then +let update_dashboard_from_json config ~json ~uid = + if uid =~! uid_rex then invalid_arg (sf "Grafana.update_dashboard: invalid UID: %s (must match: %s)" - dashboard.uid + uid (show_rex uid_rex)) ; let authorization = Option.map (fun t -> ("Authorization", "Bearer " ^ t)) config.api_token @@ -261,7 +261,7 @@ let update_dashboard config dashboard = let* () = let delete_request = { - uri = make_url config ("dashboards/uid/" ^ dashboard.uid); + uri = make_url config ("api/dashboards/uid/" ^ uid); meth = `DELETE; headers = Cohttp.Header.of_list authorization; body = None; @@ -274,12 +274,10 @@ let update_dashboard config dashboard = | status -> handle_http_error status body delete_request in (* (Re-)create dashboard. *) - let body = - `O [("dashboard", encode_dashboard config dashboard)] |> JSON.encode_u - in + let body = json in let create_request = { - uri = make_url config "dashboards/db"; + uri = make_url config "api/dashboards/db"; meth = `POST; headers = Cohttp.Header.of_list @@ -292,6 +290,12 @@ let update_dashboard config dashboard = | #Cohttp.Code.success_status -> Cohttp_lwt.Body.drain_body body | status -> handle_http_error status body create_request +let update_dashboard config dashboard = + let json = + `O [("dashboard", encode_dashboard config dashboard)] |> JSON.encode_u + in + update_dashboard_from_json config ~json ~uid:dashboard.uid + let where_clause_of_tag (tag_name, tag_label) = InfluxDB.Tag (tag_name, EQ, tag_label) diff --git a/tezt/lib_performance_regression/grafana.mli b/tezt/lib_performance_regression/grafana.mli index d637e35f0704875c23ac319e00491f03d56c2ee4..180e44f5ac0ee23a44c4d92b9de3e2089f39f6db 100644 --- a/tezt/lib_performance_regression/grafana.mli +++ b/tezt/lib_performance_regression/grafana.mli @@ -120,6 +120,11 @@ type dashboard = { @raise Invalid_arg if the dashboard UID is invalid. *) val update_dashboard : config -> dashboard -> unit Lwt.t +(** Similar to update_dashboard, except it is provided as a raw JSON + represented as a string. *) +val update_dashboard_from_json : + config -> json:string -> uid:string -> unit Lwt.t + (** Make a simple SELECT query for a graph panel. Usage: [simple_query diff --git a/tezt/lib_tezos/agent.ml b/tezt/lib_tezos/agent.ml deleted file mode 100644 index f5366cd09445e24d652e5f2ccc554b5714ddf697..0000000000000000000000000000000000000000 --- a/tezt/lib_tezos/agent.ml +++ /dev/null @@ -1,88 +0,0 @@ -(*****************************************************************************) -(* *) -(* SPDX-License-Identifier: MIT *) -(* SPDX-FileCopyrightText: 2024 Nomadic Labs *) -(* *) -(*****************************************************************************) - -type t = { - mutable name : string; - point : string * int; - runner : Runner.t; - next_available_port : unit -> int; - binaries_path : string; -} - -let make ?(ssh_user = "root") ~ssh_id ~point:((address, ssh_port) as point) - ~binaries_path ~next_available_port ~name () = - let runner = Runner.create ~ssh_user ~ssh_id ~ssh_port ~address () in - {point; runner; name; next_available_port; binaries_path} - -let name {name; _} = name - -let set_name agent name = agent.name <- name - -let path_of agent binary = agent.binaries_path // binary - -let copy agent ~source ~destination = - let runner = agent.runner in - let identity = - Option.fold ~none:[] ~some:(fun i -> ["-i"; i]) runner.ssh_id - in - let port = - Option.fold - ~none:[] - ~some:(fun p -> ["-P"; Format.sprintf "%d" p]) - runner.ssh_port - in - let* exists = - let process = Process.spawn ~runner "ls" [destination] in - let* status = process |> Process.wait in - match status with WEXITED 0 -> Lwt.return_true | _ -> Lwt.return_false - in - if exists then Lwt.return_unit - else - let destination = - Format.sprintf - "%s%s:%s" - (Option.fold - ~none:"" - ~some:(fun u -> Format.sprintf "%s@" u) - runner.ssh_user) - runner.address - destination - in - let* () = - Process.run "scp" (["-O"] @ identity @ port @ [source] @ [destination]) - in - Lwt.return_unit - -let copy = - (* We memoize the copy so that it is done at most once per destination per - scenario. This optimisation ease the writing of scenario so that copy can - always be called before using the file copied. *) - let already_copied = Hashtbl.create 11 in - fun agent ~source -> - let destination = path_of agent source in - match Hashtbl.find_opt already_copied (agent, destination) with - | Some promise -> promise - | None -> - let p = - let* () = - Process.spawn - ~runner:agent.runner - "mkdir" - ["-p"; Filename.dirname destination] - |> Process.check - in - let* () = copy agent ~source ~destination in - Lwt.return destination - in - Hashtbl.replace already_copied (agent, destination) p ; - p - -let next_available_port t = t.next_available_port () - -let runner {runner; _} = runner - -let binaries_path {binaries_path; _} = binaries_path diff --git a/tezt/lib_tezos/dune b/tezt/lib_tezos/dune index 379c0aac7bfd2e2cfad458bfb5be288a901fa590..e406f05578203133e4b1211301c12599624aa9cc 100644 --- a/tezt/lib_tezos/dune +++ b/tezt/lib_tezos/dune @@ -7,6 +7,7 @@ (libraries octez-libs.tezt-wrapper tezt-tezos.tezt-performance-regression + tezt-tezos.tezt-cloud uri hex octez-libs.crypto-dal @@ -17,4 +18,5 @@ (:standard) -open Tezt_wrapper -open Tezt_wrapper.Base - -open Tezt_tezos_tezt_performance_regression)) + -open Tezt_tezos_tezt_performance_regression + -open Tezt_tezos_tezt_cloud)) diff --git a/tezt/tests/cloud/basic.ml b/tezt/tests/cloud/basic.ml index 8754c29bbad209c68054b6e570ebe2c7fb9cc077..f3ce0cdc7622f616c6b8680d75818b7aa3542836 100644 --- a/tezt/tests/cloud/basic.ml +++ b/tezt/tests/cloud/basic.ml @@ -7,7 +7,7 @@ let simple () = Cloud.register - ~vms:[Cloud.Configuration.make (); Cloud.Configuration.make ()] + ~vms:[Configuration.make (); Configuration.make ()] ~__FILE__ ~tags:["simple"; "health"; Tag.cloud] ~title:"Simple health check to check local configuration" diff --git a/tezt/tests/cloud/dal.ml b/tezt/tests/cloud/dal.ml index c40cb02f17cc7fd2ef99d0790b6d38587dce0d26..807a0c2a6b040ffb6255d80a579d2d38c8e3a988 100644 --- a/tezt/tests/cloud/dal.ml +++ b/tezt/tests/cloud/dal.ml @@ -1451,7 +1451,9 @@ let etherlink_loop (etherlink : etherlink) = List.nth etherlink.producers (i mod List.length etherlink.producers) in let runner = Node.runner producer.node |> Option.get in - let firehose = Agent.binaries_path producer.agent // "firehose" in + let firehose = + (Agent.configuration producer.agent).binaries_path // "firehose" + in let* () = Process.spawn ~runner @@ -1511,29 +1513,38 @@ let benchmark () = ] |> List.concat in - let default_vm_configuration = Cloud.Configuration.make () in + let default_vm_configuration = Configuration.make () in let vms = vms |> List.map (function - | `Bootstrap -> default_vm_configuration + | `Bootstrap -> + (* Configuration.make ~docker_image:Octez_latest_release () *) + default_vm_configuration | `Baker i -> ( match configuration.stake_machine_type with | None -> default_vm_configuration | Some list -> ( try let machine_type = List.nth list i in - Cloud.Configuration.make ~machine_type () + Configuration.make ~machine_type () with _ -> default_vm_configuration)) | `Producer | `Observer -> ( match configuration.producer_machine_type with | None -> default_vm_configuration - | Some machine_type -> Cloud.Configuration.make ~machine_type ()) + | Some machine_type -> Configuration.make ~machine_type ()) | `Etherlink_operator -> default_vm_configuration | `Etherlink_producer _ -> default_vm_configuration) in Cloud.register (* docker images are pushed before executing the test in case binaries are modified locally. This way we always use the latest ones. *) ~vms + ~proxy_files: + [ + Format.asprintf + "src/%s/parameters/mainnet-parameters.json" + (Protocol.directory configuration.protocol); + "evm_kernel.wasm"; + ] ~__FILE__ ~title:"DAL node benchmark" ~tags:[Tag.cloud; "dal"; "benchmark"] diff --git a/tezt/tests/cloud/dune b/tezt/tests/cloud/dune index 5164063aa60b47377b340cdb48300efe7044f812..9b1145aaf882753118ea421f78a90b2fc2f55fb3 100644 --- a/tezt/tests/cloud/dune +++ b/tezt/tests/cloud/dune @@ -7,7 +7,7 @@ octez-libs.test-helpers octez-libs.tezt-wrapper tezt-tezos - tezt_cloud + tezt-tezos.tezt-cloud tezt_etherlink) (link_flags (:standard) @@ -19,4 +19,4 @@ -open Tezt_wrapper.Base -open Tezt_tezos -open Tezt_tezos.Runnable.Syntax - -open Tezt_cloud)) + -open Tezt_tezos_tezt_cloud))