From d5e49f8d8672ee2347edde2da25a779ce8814712 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 12 Feb 2025 09:46:07 +0100 Subject: [PATCH 1/4] Build: Manifest for outbox monitor binary --- Makefile | 5 ++++ manifest/product_etherlink.ml | 49 +++++++++++++++++++++++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/Makefile b/Makefile index 521188380cd5..4136a679df81 100644 --- a/Makefile +++ b/Makefile @@ -503,6 +503,11 @@ build-floodgate: @dune build ./etherlink/bin_floodgate @cp -f ./_build/default/etherlink/bin_floodgate/main.exe floodgate +.PHONY: etherlink-outbox-monitor +etherlink-outbox-monitor: + @dune build ./etherlink/bin_outbox_monitor + @cp -f ./_build/default/etherlink/bin_outbox_monitor/main.exe $@ + .PHONY: build-unreleased build-unreleased: all @echo 'Note: "make build-unreleased" is deprecated. Just use "make".' diff --git a/manifest/product_etherlink.ml b/manifest/product_etherlink.ml index fabeb54b1082..4f805a040aa5 100644 --- a/manifest/product_etherlink.ml +++ b/manifest/product_etherlink.ml @@ -477,3 +477,52 @@ let _floodgate_bin = evm_node_config |> open_; octez_workers; ] + +let _outbox_monitor = + public_exe + "etherlink-outbox-monitor" + ~internal_name:"main" + ~path:"etherlink/bin_outbox_monitor" + ~opam:"etherlink-outbox-monitor" + ~release_status:Unreleased + ~synopsis: + "A binary to monitor withdrawals in the outbox and their execution" + ~deps: + [ + bls12_381_archive; + octez_base |> open_ ~m:"TzPervasives"; + octez_base_unix; + octez_version_value; + octez_clic; + octez_rpc_http |> open_; + octez_rpc_http_client_unix; + caqti_lwt; + crunch; + re; + octez_sqlite |> open_; + evm_node_lib_dev_encoding |> open_; + ] + ~dune: + Dune. + [ + [ + S "rule"; + [S "target"; S "migrations.ml"]; + [S "deps"; [S "glob_files"; S "migrations/*.sql"]]; + [ + S "action"; + [ + S "run"; + S "ocaml-crunch"; + S "-e"; + S "sql"; + S "-m"; + S "plain"; + S "-o"; + S "%{target}"; + S "-s"; + S "."; + ]; + ]; + ]; + ] -- GitLab From d8d39dbb0ef1ebb8b526eadd83194f8a28abc8da Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 12 Feb 2025 09:46:39 +0100 Subject: [PATCH 2/4] Build: generated files --- dune-project | 1 + etherlink/bin_outbox_monitor/dune | 34 ++++++++++++++++++++++++++++++ opam/etherlink-outbox-monitor.opam | 28 ++++++++++++++++++++++++ 3 files changed, 63 insertions(+) create mode 100644 etherlink/bin_outbox_monitor/dune create mode 100644 opam/etherlink-outbox-monitor.opam diff --git a/dune-project b/dune-project index 95bcf68ed712..62873642efcb 100644 --- a/dune-project +++ b/dune-project @@ -8,6 +8,7 @@ (package (name dal_node_migrations)) (package (name efunc_core)) (package (name etherlink-governance-observer)) +(package (name etherlink-outbox-monitor)) (package (name floodgate)) (package (name gitlab_ci)) (package (name internal-devtools)) diff --git a/etherlink/bin_outbox_monitor/dune b/etherlink/bin_outbox_monitor/dune new file mode 100644 index 000000000000..660f873848a8 --- /dev/null +++ b/etherlink/bin_outbox_monitor/dune @@ -0,0 +1,34 @@ +; This file was automatically generated, do not edit. +; Edit file manifest/main.ml instead. + +(executable + (name main) + (public_name etherlink-outbox-monitor) + (package etherlink-outbox-monitor) + (instrumentation (backend bisect_ppx)) + (libraries + bls12-381.archive + octez-libs.base + octez-libs.base.unix + octez-version.value + octez-libs.clic + octez-libs.rpc-http + octez-libs.rpc-http-client-unix + caqti-lwt + re + octez-l2-libs.sqlite + octez-evm-node-libs.evm_node_lib_dev_encoding) + (link_flags + (:standard) + (:include %{workspace_root}/static-link-flags.sexp)) + (flags + (:standard) + -open Tezos_base.TzPervasives + -open Tezos_rpc_http + -open Octez_sqlite + -open Evm_node_lib_dev_encoding)) + +(rule + (target migrations.ml) + (deps (glob_files migrations/*.sql)) + (action (run ocaml-crunch -e sql -m plain -o %{target} -s .))) diff --git a/opam/etherlink-outbox-monitor.opam b/opam/etherlink-outbox-monitor.opam new file mode 100644 index 000000000000..4024e01ded83 --- /dev/null +++ b/opam/etherlink-outbox-monitor.opam @@ -0,0 +1,28 @@ +# This file was automatically generated, do not edit. +# Edit file manifest/main.ml instead. +opam-version: "2.0" +maintainer: "contact@tezos.com" +authors: ["Tezos devteam"] +homepage: "https://www.tezos.com/" +bug-reports: "https://gitlab.com/tezos/tezos/issues" +dev-repo: "git+https://gitlab.com/tezos/tezos.git" +license: "MIT" +depends: [ + "dune" { >= "3.11.1" } + "ocaml" { >= "4.14" } + "bls12-381" + "octez-libs" + "octez-version" + "caqti-lwt" { >= "2.0.1" } + "crunch" { >= "3.3.0" } + "re" { >= "1.10.0" } + "octez-l2-libs" + "octez-evm-node-libs" { = version } +] +build: [ + ["rm" "-r" "vendors" "contrib"] + ["dune" "build" "-p" name "-j" jobs] + ["dune" "runtest" "-p" name "-j" jobs] {with-test} +] +available: os-family != "windows" +synopsis: "A binary to monitor withdrawals in the outbox and their execution" -- GitLab From b3e6238af46aa1e26f2f0927ae56338257b5ce99 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Tue, 18 Feb 2025 15:13:10 +0100 Subject: [PATCH 3/4] Outbox monitor: binary entrypoint --- etherlink/bin_outbox_monitor/db.ml | 161 ++++++++++++++++++ etherlink/bin_outbox_monitor/db.mli | 14 ++ etherlink/bin_outbox_monitor/main.ml | 152 +++++++++++++++++ .../bin_outbox_monitor/sqlite_migrations.ml | 49 ++++++ 4 files changed, 376 insertions(+) create mode 100644 etherlink/bin_outbox_monitor/db.ml create mode 100644 etherlink/bin_outbox_monitor/db.mli create mode 100644 etherlink/bin_outbox_monitor/main.ml create mode 100644 etherlink/bin_outbox_monitor/sqlite_migrations.ml diff --git a/etherlink/bin_outbox_monitor/db.ml b/etherlink/bin_outbox_monitor/db.ml new file mode 100644 index 000000000000..8715b4ff57d8 --- /dev/null +++ b/etherlink/bin_outbox_monitor/db.ml @@ -0,0 +1,161 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Functori, *) +(* Copyright (c) 2025 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +open Caqti_request.Infix +open Caqti_type.Std + +(** Current version for migrations. *) +let version = 0 + +module Events = struct + include Internal_event.Simple + + let section = ["outbox_monitor"; "db"] + + let create_db = + declare_0 + ~section + ~name:"create_db" + ~msg:"Database is being created" + ~level:Info + () + + let applied_migration = + declare_1 + ~section + ~name:"applied_migration" + ~msg:"Applied migration \"{name}\" to the database" + ~level:Notice + ("name", Data_encoding.string) + + let migrations_from_the_future = + declare_2 + ~section + ~name:"migrations_from_the_future" + ~msg:"Database has {applied} migrations applied but only aware of {known}" + ~level:Error + ("applied", Data_encoding.int31) + ("known", Data_encoding.int31) +end + +let _with_connection store conn = + match conn with + | Some conn -> Sqlite.with_connection conn + | None -> + fun k -> Sqlite.use store @@ fun conn -> Sqlite.with_connection conn k + +let _with_transaction store k = + Sqlite.use store @@ fun conn -> Sqlite.with_transaction conn k + +module Migrations = struct + module Q = struct + let table_exists = + (string ->! bool) + @@ {sql| + SELECT EXISTS ( + SELECT name FROM sqlite_master + WHERE type='table' + AND name=? + )|sql} + + let create_table = + (unit ->. unit) + @@ {sql| + CREATE TABLE migrations ( + id SERIAL PRIMARY KEY, + name TEXT + )|sql} + + let current_migration = + (unit ->? int) @@ {|SELECT id FROM migrations ORDER BY id DESC LIMIT 1|} + + let register_migration = + (t2 int string ->. unit) + @@ {sql| + INSERT INTO migrations (id, name) VALUES (?, ?) + |sql} + + let all : Sqlite_migrations.migration list = + Sqlite_migrations.migrations version + end + + let create_table store = + Sqlite.with_connection store @@ fun conn -> + Sqlite.Db.exec conn Q.create_table () + + let table_exists store = + Sqlite.with_connection store @@ fun conn -> + Sqlite.Db.find conn Q.table_exists "migrations" + + let missing_migrations store = + let open Lwt_result_syntax in + let all_migrations = List.mapi (fun i m -> (i, m)) Q.all in + let* current = + Sqlite.with_connection store @@ fun conn -> + Sqlite.Db.find_opt conn Q.current_migration () + in + match current with + | Some current -> + let applied = current + 1 in + let known = List.length all_migrations in + if applied <= known then return (List.drop_n applied all_migrations) + else + let*! () = + Events.(emit migrations_from_the_future) (applied, known) + in + failwith + "Cannot use a database at migration %d, the outbox monitor only \ + supports up to %d" + applied + known + | None -> return all_migrations + + let apply_migration store id (module M : Sqlite_migrations.S) = + let open Lwt_result_syntax in + Sqlite.with_connection store @@ fun conn -> + let* () = List.iter_es (fun up -> Sqlite.Db.exec conn up ()) M.apply in + Sqlite.Db.exec conn Q.register_migration (id, M.name) +end + +type t = Sqlite.t + +let sqlite_file_name = "outbox-monitor.sqlite" + +let init ~data_dir perm : t tzresult Lwt.t = + let open Lwt_result_syntax in + let*! () = Tezos_stdlib_unix.Lwt_utils_unix.create_dir data_dir in + let path = Filename.concat data_dir sqlite_file_name in + let*! exists = Lwt_unix.file_exists path in + let migration conn = + Sqlite.assert_in_transaction conn ; + let* () = + if not exists then + let* () = Migrations.create_table conn in + let*! () = Events.(emit create_db) () in + return_unit + else + let* table_exists = Migrations.table_exists conn in + let* () = + when_ (not table_exists) (fun () -> + failwith + "A database already exists, but its content is incorrect.") + in + return_unit + in + let* migrations = Migrations.missing_migrations conn in + let* () = + List.iter_es + (fun (i, ((module M : Sqlite_migrations.S) as mig)) -> + let* () = Migrations.apply_migration conn i mig in + let*! () = Events.(emit applied_migration) M.name in + return_unit) + migrations + in + return_unit + in + Sqlite.init ~path ~perm migration diff --git a/etherlink/bin_outbox_monitor/db.mli b/etherlink/bin_outbox_monitor/db.mli new file mode 100644 index 000000000000..5b08cabb70d9 --- /dev/null +++ b/etherlink/bin_outbox_monitor/db.mli @@ -0,0 +1,14 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Functori, *) +(* Copyright (c) 2025 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +(** Data is stored in an SQLite database. *) +type t = Sqlite.t + +(** Initialize the database by creating it if it doesn't exist and applying + migrations when needed. *) +val init : data_dir:string -> [`Read_only | `Read_write] -> t tzresult Lwt.t diff --git a/etherlink/bin_outbox_monitor/main.ml b/etherlink/bin_outbox_monitor/main.ml new file mode 100644 index 000000000000..862fafa8dbd9 --- /dev/null +++ b/etherlink/bin_outbox_monitor/main.ml @@ -0,0 +1,152 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2025 Functori, *) +(* Copyright (c) 2025 Nomadic Labs, *) +(* *) +(*****************************************************************************) + +type global = {data_dir : string; verbosity : Internal_event.level} + +let default_data_dir = Filename.concat (Sys.getenv "HOME") ".outbox-monitor" + +let default_global = {data_dir = default_data_dir; verbosity = Notice} + +module Arg = struct + let verbose = + Tezos_clic.switch + ~short:'v' + ~long:"verbose" + ~doc:"Sets logging level to info" + () + + let debug = + Tezos_clic.switch ~long:"debug" ~doc:"Sets logging level to debug" () + + let data_dir = + Tezos_clic.arg + ~long:"data-dir" + ~short:'d' + ~doc: + (Format.sprintf + "Directory where data is stored for the outbox monitor.\n\ + Defaults to `%s`." + default_data_dir) + ~placeholder:"path" + @@ Tezos_clic.parameter (fun _ x -> Lwt_result.return x) +end + +let log_config ~verbosity () = + let open Tezos_base_unix.Internal_event_unix in + let config = + make_with_defaults + ~verbosity + ~log_cfg: + (Tezos_base_unix.Logs_simple_config.create_cfg + ~advertise_levels:true + ()) + () + in + init ~config () + +let global_options_args = + let open Tezos_clic in + let open Arg in + map_arg + (aggregate (args3 data_dir verbose debug)) + ~f:(fun g (data_dir, verbose, debug) -> + let verbosity = + if debug then Internal_event.Debug + else if verbose then Info + else g.verbosity + in + let data_dir = Option.value data_dir ~default:g.data_dir in + Lwt_result.return {data_dir; verbosity}) + +let global_options = Tezos_clic.args1 global_options_args + +let om_command ~desc args cmd_prefixes f = + let open Tezos_clic in + command + ~desc + (args2 global_options_args (aggregate args)) + cmd_prefixes + (fun (global, args) -> f global args) + +let run_command = + let open Tezos_clic in + om_command + ~desc:"Start monitoring outbox" + no_options + (prefixes ["run"] @@ stop) + (fun {data_dir; verbosity} () _ -> + let open Lwt_result_syntax in + let*! () = log_config ~verbosity () in + let* _db = Db.init ~data_dir `Read_write in + return_unit) + +let commands = [run_command] + +let executable_name = Filename.basename Sys.executable_name + +let dispatch args = + let open Lwt_result_syntax in + let commands = + Tezos_clic.add_manual + ~executable_name + ~global_options + (if Unix.isatty Unix.stdout then Tezos_clic.Ansi else Tezos_clic.Plain) + Format.std_formatter + commands + in + let* global, remaining_args = + Tezos_clic.parse_global_options global_options default_global args + in + Tezos_clic.dispatch commands global remaining_args + +let handle_error = function + | Ok _ -> () + | Error [Tezos_clic.Version] -> + Format.printf + "outbox-monitor.%s (%s)@." + Tezos_version_value.Current_git_info.abbreviated_commit_hash + Tezos_version_value.Current_git_info.committer_date ; + exit 0 + | Error [Tezos_clic.Help command] -> + Tezos_clic.usage + Format.std_formatter + ~executable_name + ~global_options + (match command with None -> [] | Some c -> [c]) ; + Stdlib.exit 0 + | Error errs -> + Tezos_clic.pp_cli_errors + Format.err_formatter + ~executable_name + ~global_options + ~default:Error_monad.pp + errs ; + Stdlib.exit 1 + +let argv () = Array.to_list Sys.argv |> List.tl |> Stdlib.Option.get + +let () = + Random.self_init () ; + let _ = + Tezos_clic.( + setup_formatter + Format.err_formatter + (if Unix.isatty Unix.stderr then Ansi else Plain) + Short) + in + let _ = + Tezos_clic.( + setup_formatter + Format.std_formatter + (if Unix.isatty Unix.stdout then Ansi else Plain) + Short) + in + Lwt.Exception_filter.(set handle_all_except_runtime) ; + Tezos_base_unix.Event_loop.main_run + (Lwt_exit.wrap_and_exit (dispatch (argv ()))) + |> handle_error diff --git a/etherlink/bin_outbox_monitor/sqlite_migrations.ml b/etherlink/bin_outbox_monitor/sqlite_migrations.ml new file mode 100644 index 000000000000..9f2b82ca072a --- /dev/null +++ b/etherlink/bin_outbox_monitor/sqlite_migrations.ml @@ -0,0 +1,49 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2024 Nomadic Labs *) +(* *) +(*****************************************************************************) + +open Caqti_request.Infix +open Caqti_type.Std + +let re_t = Re.(compile @@ Posix.re "[0-9]{3}_([a-z0-9_]+).sql") + +module type S = sig + val name : string + + val apply : (unit, unit, [`Zero]) Caqti_request.t list +end + +type migration = (module S) + +let make_migration path = + let read_exn path = + match Migrations.read path with + | Some content -> content + | None -> raise (Invalid_argument "read_exn") + in + + let migration_name = + let open Re in + let (group : Group.t) = exec re_t path in + Group.get group 1 + in + + let migration_step = ( @@ ) (unit ->. unit) in + + let make_migration_requests str = + String.split ';' str + |> List.filter_map (fun i -> + match String.trim i with "" -> None | x -> Some (migration_step x)) + in + + (module struct + let name = migration_name + + let apply = make_migration_requests (read_exn path) + end : S) + +let migrations version = + List.take_n (version + 1) Migrations.file_list |> List.map make_migration -- GitLab From 0ecf0083946e1b05760358ebda2e06810f160938 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Thu, 13 Feb 2025 11:16:22 +0100 Subject: [PATCH 4/4] gitignore new binary for outbox monitor --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e0c64990ea21..7791bba82be5 100644 --- a/.gitignore +++ b/.gitignore @@ -130,6 +130,7 @@ _evm_unstripped_installer_preimages/ /etherlink/kernels-* etherlink-governance-observer floodgate +etherlink-outbox-monitor # Terraform -- GitLab