From 9dfaef509c309f47cbaa6fed077e9e5dd616caac Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 8 Nov 2023 17:26:31 +0100 Subject: [PATCH 1/9] Manifest: rollup node uses tar library for snapshots --- manifest/main.ml | 3 +++ opam/octez-smart-rollup-node-lib.opam | 3 +++ src/lib_smart_rollup_node/dune | 3 +++ 3 files changed, 9 insertions(+) diff --git a/manifest/main.ml b/manifest/main.ml index 3ba6e44ba710..fd2b5293588a 100644 --- a/manifest/main.ml +++ b/manifest/main.ml @@ -4585,6 +4585,9 @@ let octez_smart_rollup_node_lib = cohttp_lwt_unix; octez_node_config; prometheus_app; + camlzip; + tar; + tar_unix; octez_dal_node_lib |> open_; octez_dac_lib |> open_; octez_dac_client_lib |> open_; diff --git a/opam/octez-smart-rollup-node-lib.opam b/opam/octez-smart-rollup-node-lib.opam index 60b753f1107b..b0ed900c38b6 100644 --- a/opam/octez-smart-rollup-node-lib.opam +++ b/opam/octez-smart-rollup-node-lib.opam @@ -15,6 +15,9 @@ depends: [ "cohttp-lwt-unix" { >= "5.2.0" } "octez-node-config" "prometheus-app" { >= "1.2" } + "camlzip" { >= "1.11" & < "1.12" } + "tar" + "tar-unix" { >= "2.0.1" & < "3.0.0" } "tezos-dal-node-lib" "tezos-dac-lib" "tezos-dac-client-lib" diff --git a/src/lib_smart_rollup_node/dune b/src/lib_smart_rollup_node/dune index 52bf47a0fd78..4593b1d08632 100644 --- a/src/lib_smart_rollup_node/dune +++ b/src/lib_smart_rollup_node/dune @@ -15,6 +15,9 @@ cohttp-lwt-unix octez-node-config prometheus-app + camlzip + tar + tar-unix tezos-dal-node-lib tezos-dac-lib tezos-dac-client-lib -- GitLab From 588fa01a4a30846e6ad27e959ab65e83252e5f29 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 8 Nov 2023 20:20:12 +0100 Subject: [PATCH 2/9] Rollup node: utilities for creating archives --- src/lib_smart_rollup_node/snapshot_utils.ml | 183 +++++++++++++++++++ src/lib_smart_rollup_node/snapshot_utils.mli | 48 +++++ 2 files changed, 231 insertions(+) create mode 100644 src/lib_smart_rollup_node/snapshot_utils.ml create mode 100644 src/lib_smart_rollup_node/snapshot_utils.mli diff --git a/src/lib_smart_rollup_node/snapshot_utils.ml b/src/lib_smart_rollup_node/snapshot_utils.ml new file mode 100644 index 000000000000..8ade2f192a0a --- /dev/null +++ b/src/lib_smart_rollup_node/snapshot_utils.ml @@ -0,0 +1,183 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Functori *) +(* *) +(*****************************************************************************) + +module type READER = sig + type in_channel + + val open_in : string -> in_channel + + val really_input : in_channel -> bytes -> int -> int -> unit + + val input : in_channel -> bytes -> int -> int -> int + + val close_in : in_channel -> unit +end + +module type WRITER = sig + type out_channel + + val open_out : string -> out_channel + + val output : out_channel -> bytes -> int -> int -> unit + + val flush_continue : out_channel -> unit + + val close_out : out_channel -> unit +end + +module Stdlib_reader : READER with type in_channel = Stdlib.in_channel = Stdlib + +module Stdlib_writer : WRITER with type out_channel = Stdlib.out_channel = +struct + include Stdlib + + let flush_continue = flush +end + +module Gzip_reader : READER with type in_channel = Gzip.in_channel = Gzip + +module Gzip_writer : WRITER with type out_channel = Gzip.out_channel = struct + include Gzip + + let open_out f = open_out f +end + +type reader = (module READER) + +type writer = (module WRITER) + +let stdlib_reader : reader = (module Stdlib_reader) + +let stdlib_writer : writer = (module Stdlib_writer) + +let gzip_reader : reader = (module Gzip_reader) + +let gzip_writer : writer = (module Gzip_writer) + +let list_files dir ~include_file f = + let rec list_files_in_dir stream + ((dir, relative_dir, dir_handle) as current_dir_info) = + match Unix.readdir dir_handle with + | "." | ".." -> list_files_in_dir stream current_dir_info + | basename -> + let full_path = Filename.concat dir basename in + let relative_path = Filename.concat relative_dir basename in + let stream = + if Sys.is_directory full_path then + let sub_dir_handle = Unix.opendir full_path in + list_files_in_dir stream (full_path, relative_path, sub_dir_handle) + else if include_file ~relative_path then + Stream.icons (f ~full_path ~relative_path) stream + else stream + in + list_files_in_dir stream current_dir_info + | exception End_of_file -> + Unix.closedir dir_handle ; + stream + in + let dir_handle = Unix.opendir dir in + list_files_in_dir Stream.sempty (dir, "", dir_handle) + +let create (module Reader : READER) (module Writer : WRITER) ~dir ~include_file + ~dest = + let module Archive_writer = Tar.Make (struct + include Reader + include Writer + end) in + let write_file file (out_chan : Writer.out_channel) = + let in_chan = Reader.open_in file in + try + let buffer_size = 64 * 1024 in + let buf = Bytes.create buffer_size in + let rec copy () = + let read_bytes = Reader.input in_chan buf 0 buffer_size in + Writer.output out_chan buf 0 read_bytes ; + if read_bytes > 0 then copy () + in + copy () ; + Writer.flush_continue out_chan ; + Reader.close_in in_chan + with e -> + Reader.close_in in_chan ; + raise e + in + let file_stream = + list_files dir ~include_file @@ fun ~full_path ~relative_path -> + let {Unix.st_perm; st_size; st_mtime; _} = Unix.lstat full_path in + let header = + Tar.Header.make + ~file_mode:st_perm + ~mod_time:(Int64.of_float st_mtime) + relative_path + (Int64.of_int st_size) + in + let writer = write_file full_path in + (header, writer) + in + let out_chan = Writer.open_out dest in + try + Archive_writer.Archive.create_gen file_stream out_chan ; + Writer.close_out out_chan + with e -> + Writer.close_out out_chan ; + raise e + +let rec create_dir ?(perm = 0o755) dir = + let stat = + try Some (Unix.stat dir) with Unix.Unix_error (ENOENT, _, _) -> None + in + match stat with + | Some {st_kind = S_DIR; _} -> () + | Some _ -> Stdlib.failwith "Not a directory" + | None -> ( + create_dir ~perm (Filename.dirname dir) ; + try Unix.mkdir dir perm + with Unix.Unix_error (EEXIST, _, _) -> + (* This is the case where the directory has been created at the same + time. *) + ()) + +let extract (module Reader : READER) (module Writer : WRITER) ~snapshot_file + ~dest = + let module Archive_reader = Tar.Make (struct + include Reader + include Writer + end) in + let out_channel_of_header (header : Tar.Header.t) = + let path = Filename.concat dest header.file_name in + create_dir (Filename.dirname path) ; + Writer.open_out path + in + let in_chan = Reader.open_in snapshot_file in + try + Archive_reader.Archive.extract_gen out_channel_of_header in_chan ; + Reader.close_in in_chan + with e -> + Reader.close_in in_chan ; + raise e + +let compress ~snapshot_file = + let snapshot_file_gz = Filename.chop_suffix snapshot_file ".uncompressed" in + let in_chan = open_in snapshot_file in + let out_chan = Gzip.open_out snapshot_file_gz in + try + let buffer_size = 64 * 1024 in + let buf = Bytes.create buffer_size in + let rec copy () = + let read_bytes = input in_chan buf 0 buffer_size in + Gzip.output out_chan buf 0 read_bytes ; + if read_bytes > 0 then copy () + in + copy () ; + Gzip.close_out out_chan ; + close_in in_chan ; + Unix.unlink snapshot_file ; + snapshot_file_gz + with e -> + Gzip.close_out out_chan ; + close_in in_chan ; + raise e diff --git a/src/lib_smart_rollup_node/snapshot_utils.mli b/src/lib_smart_rollup_node/snapshot_utils.mli new file mode 100644 index 000000000000..d851e6221c0a --- /dev/null +++ b/src/lib_smart_rollup_node/snapshot_utils.mli @@ -0,0 +1,48 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Functori *) +(* *) +(*****************************************************************************) + +(** {2 Snapshot archives creation and extraction} *) + +(** The type of snapshot archive readers. *) +type reader + +(** The type of snapshot archive writers. *) +type writer + +(** A reader for uncompressed files or snapshot archives. *) +val stdlib_reader : reader + +(** A writer for uncompressed files or snapshot archives. *) +val stdlib_writer : writer + +(** A reader for compressed files or snapshot archives. *) +val gzip_reader : reader + +(** A writer for compressed files or snapshot archives. *) +val gzip_writer : writer + +(** [create reader writer ~dir ~include_file ~dest] creates a snapshot archive + with the hierarchy of files in directory [dir] for which [include_file] + returns true. The archive is produced in file [dest]. *) +val create : + reader -> + writer -> + dir:string -> + include_file:(relative_path:string -> bool) -> + dest:string -> + unit + +(** [extract reader writer ~snapshot_file ~dest] extracts the snapshot archive + [snapshot_file] in the directory [dest]. Existing files in [dest] with the + same names are overwritten. *) +val extract : reader -> writer -> snapshot_file:string -> dest:string -> unit + +(** [compress ~snapshot_file] compresses the snapshot archive [snapshot_file] of + the form ["path/to/snapshot.uncompressed"] to a new file + ["path/to/snapshot"] whose path is returned. [snapshot_file] is removed upon + successful compression. *) +val compress : snapshot_file:string -> string -- GitLab From c5e6f5bc281e32a0a9559f9c890f5ca8dc8a4ee2 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 8 Nov 2023 20:37:50 +0100 Subject: [PATCH 3/9] Rollup node: snapshot export --- src/lib_smart_rollup_node/snapshots.ml | 125 ++++++++++++++++++++++++ src/lib_smart_rollup_node/snapshots.mli | 11 +++ 2 files changed, 136 insertions(+) create mode 100644 src/lib_smart_rollup_node/snapshots.ml create mode 100644 src/lib_smart_rollup_node/snapshots.mli diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml new file mode 100644 index 000000000000..2fb404ba7f32 --- /dev/null +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -0,0 +1,125 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Functori *) +(* *) +(*****************************************************************************) + +open Snapshot_utils + +let check_store_version store_dir = + let open Lwt_result_syntax in + let* store_version = Store_version.read_version_file ~dir:store_dir in + let*? () = + match store_version with + | None -> error_with "Unversionned store, cannot produce snapshot." + | Some v when v <> Store.version -> + error_with + "Incompatible store version %a, expected %a. Cannot produce \ + snapshot. Please restart your rollup node to migrate." + Store_version.pp + v + Store_version.pp + Store.version + | Some _ -> Ok () + in + return_unit + +let check_head (store : _ Store.t) context = + let open Lwt_result_syntax in + let* head = Store.L2_head.read store.l2_head in + let*? head = + match head with + | None -> + error_with + "There is no head in the rollup node store, cannot produce snapshot." + | Some head -> Ok head + in + (* Ensure head context is available. *) + let*! head_ctxt = Context.checkout context head.header.context in + let*? () = + error_when (Option.is_none head_ctxt) + @@ error_of_fmt "Head context cannot be checkouted, won't produce snapshot." + in + return head + +let pre_export_checks_and_get_snapshot_metadata ~data_dir = + let open Lwt_result_syntax in + let store_dir = Configuration.default_storage_dir data_dir in + let context_dir = Configuration.default_context_dir data_dir in + (* Load context and stores in read-only to check they are valid. *) + let* () = check_store_version store_dir in + let* metadata = Metadata.read_metadata_file ~dir:data_dir in + let*? metadata = + match metadata with + | None -> error_with "No rollup node metadata in %S." data_dir + | Some m -> Ok m + in + let*? () = Context.Version.check metadata.context_version in + let* context = Context.load ~cache_size:1 Read_only context_dir in + let* store = + Store.load Read_only ~index_buffer_size:0 ~l2_blocks_cache_size:1 store_dir + in + let* history_mode = Store.History_mode.read store.history_mode in + let*? history_mode = + match history_mode with + | None -> error_with "No history mode information in %S." data_dir + | Some h -> Ok h + in + let* head = check_head store context in + (* Closing context and stores after checks *) + let*! () = Context.close context in + let* () = Store.close store in + return (history_mode, metadata.rollup_address, head.header.level) + +let operator_local_file_regexp = + Re.Str.regexp "^storage/\\(commitments_published_at_level.*\\|lpc$\\)" + +let snapshotable_files_regexp = + Re.Str.regexp + "^\\(storage/.*\\|context/.*\\|wasm_2_0_0/.*\\|arith/.*\\|context/.*\\|metadata$\\)" + +let export ~data_dir ~dest = + let open Lwt_result_syntax in + let* uncompressed_snapshot = + Format.eprintf "Acquiring GC lock@." ; + (* Take GC lock first in order to not prevent progression of rollup node. *) + Utils.with_lockfile (Node_context.gc_lockfile_path ~data_dir) @@ fun () -> + Format.eprintf "Acquiring process lock@." ; + Utils.with_lockfile (Node_context.processing_lockfile_path ~data_dir) + @@ fun () -> + let* history_mode, address, head_level = + pre_export_checks_and_get_snapshot_metadata ~data_dir + in + let dest_file_name = + Format.asprintf + "snapshot-%a-%ld.%s.uncompressed" + Address.pp_short + address + head_level + (Configuration.string_of_history_mode history_mode) + in + let dest_file = + match dest with + | Some dest -> Filename.concat dest dest_file_name + | None -> dest_file_name + in + let*! () = + let open Lwt_syntax in + let* () = Option.iter_s Lwt_utils_unix.create_dir dest in + let include_file ~relative_path = + Re.Str.string_match snapshotable_files_regexp relative_path 0 + && not (Re.Str.string_match operator_local_file_regexp relative_path 0) + in + create + stdlib_reader + stdlib_writer + ~dir:data_dir + ~include_file + ~dest:dest_file ; + return_unit + in + return dest_file + in + let snapshot_file = compress ~snapshot_file:uncompressed_snapshot in + return snapshot_file diff --git a/src/lib_smart_rollup_node/snapshots.mli b/src/lib_smart_rollup_node/snapshots.mli new file mode 100644 index 000000000000..a365907dbe9a --- /dev/null +++ b/src/lib_smart_rollup_node/snapshots.mli @@ -0,0 +1,11 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* Copyright (c) 2023 Functori *) +(* *) +(*****************************************************************************) + +(** [export ~data_dir ~dest] creates a tar gzipped archive in [dest] (or the + current directory) containing a snapshot of the data of the rollup node with + data directory [data_dir]. The path of the snapshot archive is returned. *) +val export : data_dir:string -> dest:string option -> string tzresult Lwt.t -- GitLab From 79a83bec66452e5731b1ef97d481303b03e8a6a7 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 8 Nov 2023 17:28:15 +0100 Subject: [PATCH 4/9] Rollup node: command for snapshot export --- .../main_smart_rollup_node.ml | 14 ++++++++++++++ src/lib_smart_rollup_node/cli.ml | 9 +++++++++ 2 files changed, 23 insertions(+) diff --git a/src/bin_smart_rollup_node/main_smart_rollup_node.ml b/src/bin_smart_rollup_node/main_smart_rollup_node.ml index dff4bc3698b9..7069a3ceda36 100644 --- a/src/bin_smart_rollup_node/main_smart_rollup_node.ml +++ b/src/bin_smart_rollup_node/main_smart_rollup_node.ml @@ -374,6 +374,19 @@ let dump_durable_storage = return_unit | Error errs -> cctxt#error "%a" pp_print_trace errs) +let export_snapshot = + let open Tezos_clic in + command + ~group + ~desc:"Export a snapshot of the rollup node state." + (args2 data_dir_arg Cli.snapshot_dir_arg) + (prefixes ["snapshot"; "export"] @@ stop) + (fun (data_dir, dest) cctxt -> + let open Lwt_result_syntax in + let* snapshot_file = Snapshots.export ~data_dir ~dest in + let*! () = cctxt#message "Snapshot exported to %s@." snapshot_file in + return_unit) + let sc_rollup_commands () = [ config_init_command; @@ -382,6 +395,7 @@ let sc_rollup_commands () = protocols_command; dump_metrics; dump_durable_storage; + export_snapshot; ] let select_commands _ctxt _ = Lwt_result_syntax.return (sc_rollup_commands ()) diff --git a/src/lib_smart_rollup_node/cli.ml b/src/lib_smart_rollup_node/cli.ml index ad196a16d78d..2d7ed6be4253 100644 --- a/src/lib_smart_rollup_node/cli.ml +++ b/src/lib_smart_rollup_node/cli.ml @@ -397,6 +397,15 @@ let wasm_dump_file_param next = string_parameter next +let snapshot_dir_arg = + Tezos_clic.arg + ~long:"dest" + ~placeholder:"path" + ~doc: + "Directory in which to export the snapshot (defaults to current \ + directory)" + string_parameter + let string_list = Tezos_clic.parameter (fun (_cctxt : Client_context.full) s -> let list = String.split ',' s in -- GitLab From 7343929b89680e9d81d436cf6621c0c95c2809e7 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 8 Nov 2023 14:39:01 +0100 Subject: [PATCH 5/9] Test: snapshot exports --- tezt/lib_tezos/sc_rollup_node.ml | 21 ++++++++++++++ tezt/lib_tezos/sc_rollup_node.mli | 4 +++ tezt/tests/sc_rollup.ml | 48 +++++++++++++++++++++++++++++++ 3 files changed, 73 insertions(+) diff --git a/tezt/lib_tezos/sc_rollup_node.ml b/tezt/lib_tezos/sc_rollup_node.ml index 03fc1e8cb0db..c7ed100b10ca 100644 --- a/tezt/lib_tezos/sc_rollup_node.ml +++ b/tezt/lib_tezos/sc_rollup_node.ml @@ -608,6 +608,27 @@ let dump_durable_storage ~sc_rollup_node ~dump ?(block = "head") () = let process = spawn_command sc_rollup_node cmd in Process.check process +let export_snapshot sc_rollup_node dir = + let process = + spawn_command + sc_rollup_node + [ + "snapshot"; + "export"; + "--dest"; + dir; + "--data-dir"; + data_dir sc_rollup_node; + ] + in + let parse process = + let* output = Process.check_and_read_stdout process in + match output =~* rex "Snapshot exported to ([^\n]*)" with + | None -> Test.fail "Snapshot export failed" + | Some filename -> return filename + in + Runnable.{value = process; run = parse} + let as_rpc_endpoint (t : t) = let state = t.persistent_state in let scheme = "http" in diff --git a/tezt/lib_tezos/sc_rollup_node.mli b/tezt/lib_tezos/sc_rollup_node.mli index 4106d74b47f1..cdb5c66a5bec 100644 --- a/tezt/lib_tezos/sc_rollup_node.mli +++ b/tezt/lib_tezos/sc_rollup_node.mli @@ -318,6 +318,10 @@ val change_node_mode : t -> mode -> t val dump_durable_storage : sc_rollup_node:t -> dump:string -> ?block:string -> unit -> unit Lwt.t +(** [export_snapshot rollup_node dir] creates a snapshot of the rollup node in + directory [dir]. *) +val export_snapshot : t -> string -> string Runnable.process + (** Expose the RPC server address of this node as a foreign endpoint. *) val as_rpc_endpoint : t -> Endpoint.t diff --git a/tezt/tests/sc_rollup.ml b/tezt/tests/sc_rollup.ml index 596800b59564..595b1e856696 100644 --- a/tezt/tests/sc_rollup.ml +++ b/tezt/tests/sc_rollup.ml @@ -1010,6 +1010,42 @@ let test_gc variant ~challenge_window ~commitment_period ~history_mode = | _ -> ()) ; unit +(* Testing that snapshots can be exported correctly for a running node. *) +let test_snapshots ~challenge_window ~commitment_period ~history_mode = + let history_mode_str = Sc_rollup_node.string_of_history_mode history_mode in + test_full_scenario + { + tags = ["snapshot"; history_mode_str]; + variant = None; + description = + sf "snapshot can be exported and checked (%s)" history_mode_str; + } + ~challenge_window + ~commitment_period + @@ fun _protocol sc_rollup_node _rollup_client sc_rollup _node client -> + (* We want to produce snapshots for rollup node which have cemented + commitments *) + let level_snapshot = 2 * challenge_window in + (* We want to build an L2 chain that goes beyond the snapshots (and has + additional commitments). *) + let total_blocks = level_snapshot + (4 * commitment_period) in + let* () = Sc_rollup_node.run ~history_mode sc_rollup_node sc_rollup [] in + let rollup_node_processing = + let* () = bake_levels total_blocks client in + let* (_ : int) = Sc_rollup_node.wait_sync sc_rollup_node ~timeout:3. in + unit + in + let* (_ : int) = + Sc_rollup_node.wait_for_level sc_rollup_node level_snapshot + in + let dir = Tezt.Temp.dir "snapshots" in + let*! snapshot_path = Sc_rollup_node.export_snapshot sc_rollup_node dir in + let* exists = Lwt_unix.file_exists snapshot_path in + if not exists then + Test.fail ~__LOC__ "Snapshot file %s does not exist" snapshot_path ; + let* () = rollup_node_processing in + unit + (* One can retrieve the list of originated SCORUs. ----------------------------------------------- *) @@ -5522,6 +5558,18 @@ let register ~kind ~protocols = ~commitment_period:5 ~history_mode:Archive protocols ; + test_snapshots + ~kind + ~challenge_window:5 + ~commitment_period:2 + ~history_mode:Full + protocols ; + test_snapshots + ~kind + ~challenge_window:10 + ~commitment_period:5 + ~history_mode:Archive + protocols ; test_rpcs ~kind protocols ; test_rollup_inbox_of_rollup_node ~kind -- GitLab From 5ca886d32b3794641a14c447f4ebea4edb8ce430 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Wed, 8 Nov 2023 21:13:37 +0100 Subject: [PATCH 6/9] Rollup node: write snapshot metadata header in archive --- src/lib_smart_rollup_node/snapshot_utils.ml | 93 +++++++++++++++++++- src/lib_smart_rollup_node/snapshot_utils.mli | 36 ++++++-- src/lib_smart_rollup_node/snapshots.ml | 19 ++-- 3 files changed, 130 insertions(+), 18 deletions(-) diff --git a/src/lib_smart_rollup_node/snapshot_utils.ml b/src/lib_smart_rollup_node/snapshot_utils.ml index 8ade2f192a0a..9f3bbae3e298 100644 --- a/src/lib_smart_rollup_node/snapshot_utils.ml +++ b/src/lib_smart_rollup_node/snapshot_utils.ml @@ -29,6 +29,18 @@ module type WRITER = sig val close_out : out_channel -> unit end +module type READER_INPUT = sig + include READER + + val in_chan : in_channel +end + +module type WRITER_OUTPUT = sig + include WRITER + + val out_chan : out_channel +end + module Stdlib_reader : READER with type in_channel = Stdlib.in_channel = Stdlib module Stdlib_writer : WRITER with type out_channel = Stdlib.out_channel = @@ -58,6 +70,63 @@ let gzip_reader : reader = (module Gzip_reader) let gzip_writer : writer = (module Gzip_writer) +type snapshot_version = V0 + +type snapshot_metadata = { + history_mode : Configuration.history_mode; + address : Address.t; + head_level : int32; + last_commitment : Commitment.Hash.t; +} + +let snapshot_version_encoding = + let open Data_encoding in + conv_with_guard + (function V0 -> 0) + (function + | 0 -> Ok V0 | x -> Error ("Invalid snapshot version " ^ string_of_int x)) + int8 + +let snaphsot_metadata_encoding = + let open Data_encoding in + conv + (fun {history_mode; address; head_level; last_commitment} -> + (history_mode, address, head_level, last_commitment)) + (fun (history_mode, address, head_level, last_commitment) -> + {history_mode; address; head_level; last_commitment}) + @@ obj4 + (req "history_mode" Configuration.history_mode_encoding) + (req "address" Address.encoding) + (req "head_level" int32) + (req "last_commitment" Commitment.Hash.encoding) + +let snapshot_metadata_size = + Data_encoding.Binary.fixed_length snaphsot_metadata_encoding + |> WithExceptions.Option.get ~loc:__LOC__ + +let version = V0 + +let write_snapshot_metadata (module Writer : WRITER_OUTPUT) metadata = + let version_bytes = + Data_encoding.Binary.to_bytes_exn snapshot_version_encoding version + in + let metadata_bytes = + Data_encoding.Binary.to_bytes_exn snaphsot_metadata_encoding metadata + in + Writer.output Writer.out_chan version_bytes 0 (Bytes.length version_bytes) ; + Writer.output Writer.out_chan metadata_bytes 0 (Bytes.length metadata_bytes) + +let read_snapshot_metadata (module Reader : READER_INPUT) = + let version_bytes = Bytes.create 1 in + let metadata_bytes = Bytes.create snapshot_metadata_size in + Reader.really_input Reader.in_chan version_bytes 0 1 ; + Reader.really_input Reader.in_chan metadata_bytes 0 snapshot_metadata_size ; + let snapshot_version = + Data_encoding.Binary.of_bytes_exn snapshot_version_encoding version_bytes + in + assert (snapshot_version = version) ; + Data_encoding.Binary.of_bytes_exn snaphsot_metadata_encoding metadata_bytes + let list_files dir ~include_file f = let rec list_files_in_dir stream ((dir, relative_dir, dir_handle) as current_dir_info) = @@ -82,8 +151,8 @@ let list_files dir ~include_file f = let dir_handle = Unix.opendir dir in list_files_in_dir Stream.sempty (dir, "", dir_handle) -let create (module Reader : READER) (module Writer : WRITER) ~dir ~include_file - ~dest = +let create (module Reader : READER) (module Writer : WRITER) metadata ~dir + ~include_file ~dest = let module Archive_writer = Tar.Make (struct include Reader include Writer @@ -120,6 +189,13 @@ let create (module Reader : READER) (module Writer : WRITER) ~dir ~include_file in let out_chan = Writer.open_out dest in try + write_snapshot_metadata + (module struct + include Writer + + let out_chan = out_chan + end) + metadata ; Archive_writer.Archive.create_gen file_stream out_chan ; Writer.close_out out_chan with e -> @@ -141,8 +217,8 @@ let rec create_dir ?(perm = 0o755) dir = time. *) ()) -let extract (module Reader : READER) (module Writer : WRITER) ~snapshot_file - ~dest = +let extract (module Reader : READER) (module Writer : WRITER) metadata_check + ~snapshot_file ~dest = let module Archive_reader = Tar.Make (struct include Reader include Writer @@ -154,6 +230,15 @@ let extract (module Reader : READER) (module Writer : WRITER) ~snapshot_file in let in_chan = Reader.open_in snapshot_file in try + let metadata = + read_snapshot_metadata + (module struct + include Reader + + let in_chan = in_chan + end) + in + metadata_check metadata ; Archive_reader.Archive.extract_gen out_channel_of_header in_chan ; Reader.close_in in_chan with e -> diff --git a/src/lib_smart_rollup_node/snapshot_utils.mli b/src/lib_smart_rollup_node/snapshot_utils.mli index d851e6221c0a..942e81724716 100644 --- a/src/lib_smart_rollup_node/snapshot_utils.mli +++ b/src/lib_smart_rollup_node/snapshot_utils.mli @@ -25,21 +25,43 @@ val gzip_reader : reader (** A writer for compressed files or snapshot archives. *) val gzip_writer : writer -(** [create reader writer ~dir ~include_file ~dest] creates a snapshot archive - with the hierarchy of files in directory [dir] for which [include_file] - returns true. The archive is produced in file [dest]. *) +(** Versioning of snapshot format. Only one version for now. *) +type snapshot_version = V0 + +(** Snapshot metadata for version 0. This information is written as a header of + the archive snapshot file. *) +type snapshot_metadata = { + history_mode : Configuration.history_mode; + address : Address.t; + head_level : int32; + last_commitment : Commitment.Hash.t; +} + +(** [create reader writer metadata ~dir ~include_file ~dest] creates a snapshot + archive with the header [metadata] and the hierarchy of files in directory + [dir] for which [include_file] returns true. The archive is produced in file + [dest]. *) val create : reader -> writer -> + snapshot_metadata -> dir:string -> include_file:(relative_path:string -> bool) -> dest:string -> unit -(** [extract reader writer ~snapshot_file ~dest] extracts the snapshot archive - [snapshot_file] in the directory [dest]. Existing files in [dest] with the - same names are overwritten. *) -val extract : reader -> writer -> snapshot_file:string -> dest:string -> unit +(** [extract reader writer check_metadata ~snapshot_file ~dest] extracts the + snapshot archive [snapshot_file] in the directory [dest]. Existing files in + [dest] with the same names are overwritten. The metadata header read from + the snapshot is checked with [check_metadata] before beginning + extraction. *) +val extract : + reader -> + writer -> + (snapshot_metadata -> unit) -> + snapshot_file:string -> + dest:string -> + unit (** [compress ~snapshot_file] compresses the snapshot archive [snapshot_file] of the form ["path/to/snapshot.uncompressed"] to a new file diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index 2fb404ba7f32..4852cfbb6e96 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -70,7 +70,13 @@ let pre_export_checks_and_get_snapshot_metadata ~data_dir = (* Closing context and stores after checks *) let*! () = Context.close context in let* () = Store.close store in - return (history_mode, metadata.rollup_address, head.header.level) + return + { + history_mode; + address = metadata.rollup_address; + head_level = head.header.level; + last_commitment = Sc_rollup_block.most_recent_commitment head.header; + } let operator_local_file_regexp = Re.Str.regexp "^storage/\\(commitments_published_at_level.*\\|lpc$\\)" @@ -88,16 +94,14 @@ let export ~data_dir ~dest = Format.eprintf "Acquiring process lock@." ; Utils.with_lockfile (Node_context.processing_lockfile_path ~data_dir) @@ fun () -> - let* history_mode, address, head_level = - pre_export_checks_and_get_snapshot_metadata ~data_dir - in + let* metadata = pre_export_checks_and_get_snapshot_metadata ~data_dir in let dest_file_name = Format.asprintf "snapshot-%a-%ld.%s.uncompressed" Address.pp_short - address - head_level - (Configuration.string_of_history_mode history_mode) + metadata.address + metadata.head_level + (Configuration.string_of_history_mode metadata.history_mode) in let dest_file = match dest with @@ -114,6 +118,7 @@ let export ~data_dir ~dest = create stdlib_reader stdlib_writer + metadata ~dir:data_dir ~include_file ~dest:dest_file ; -- GitLab From c1efc4a417afbc756842463d6686ec8a57a80a48 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Fri, 1 Dec 2023 14:50:37 +0100 Subject: [PATCH 7/9] Rollup node: check snapshot integrity after export --- src/lib_smart_rollup_node/snapshots.ml | 71 ++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index 4852cfbb6e96..ce7fd9721292 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -78,6 +78,76 @@ let pre_export_checks_and_get_snapshot_metadata ~data_dir = last_commitment = Sc_rollup_block.most_recent_commitment head.header; } +let first_available_level ~data_dir store = + let open Lwt_result_syntax in + let* gc_levels = Store.Gc_levels.read store.Store.gc_levels in + match gc_levels with + | Some {first_available_level; _} -> return first_available_level + | None -> ( + let* metadata = Metadata.read_metadata_file ~dir:data_dir in + match metadata with + | None -> failwith "No metadata (needs rollup genesis info)." + | Some {genesis_info = {level; _}; _} -> return level) + +let check_some hash what = function + | Some x -> Ok x + | None -> + error_with "Could not read %s at %a after export." what Block_hash.pp hash + +let check_l2_chain ~data_dir (store : _ Store.t) context + (head : Sc_rollup_block.t) = + let open Lwt_result_syntax in + let* first_available_level = first_available_level ~data_dir store in + let rec check_block hash = + let* b = Store.L2_blocks.read store.l2_blocks hash in + let*? _b, header = check_some hash "L2 block" b in + let* messages = Store.Messages.read store.messages header.inbox_witness in + let*? _messages = check_some hash "messages" messages in + let* inbox = Store.Inboxes.read store.inboxes header.inbox_hash in + let*? _inbox = check_some hash "inbox" inbox in + let* () = + match header.commitment_hash with + | None -> return_unit + | Some commitment_hash -> + let* commitment = + Store.Commitments.read store.commitments commitment_hash + in + let*? _commitment = check_some hash "commitment" commitment in + return_unit + in + (* Ensure head context is available. *) + let*! head_ctxt = Context.checkout context header.context in + let*? _head_ctxt = check_some hash "context" head_ctxt in + if header.level <= first_available_level then return_unit + else check_block header.predecessor + in + check_block head.header.block_hash + +let post_import_checks ~dest = + let open Lwt_result_syntax in + let store_dir = Configuration.default_storage_dir dest in + let context_dir = Configuration.default_context_dir dest in + (* Load context and stores in read-only to run checks. *) + let* () = check_store_version store_dir in + let* context = Context.load ~cache_size:100 Read_only context_dir in + let* store = + Store.load + Read_only + ~index_buffer_size:1000 + ~l2_blocks_cache_size:100 + store_dir + in + let* head = check_head store context in + let* () = check_l2_chain ~data_dir:dest store context head in + let*! () = Context.close context in + let* () = Store.close store in + return_unit + +let post_export_checks ~snapshot_file = + Lwt_utils_unix.with_tempdir "snapshot_checks_" @@ fun dest -> + extract gzip_reader stdlib_writer (fun _ -> ()) ~snapshot_file ~dest ; + post_import_checks ~dest + let operator_local_file_regexp = Re.Str.regexp "^storage/\\(commitments_published_at_level.*\\|lpc$\\)" @@ -127,4 +197,5 @@ let export ~data_dir ~dest = return dest_file in let snapshot_file = compress ~snapshot_file:uncompressed_snapshot in + let* () = post_export_checks ~snapshot_file in return snapshot_file -- GitLab From 01a26aa4c66b9fe1ebc9083985cab7636352e129 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Fri, 1 Dec 2023 14:52:15 +0100 Subject: [PATCH 8/9] Rollup node: display progress for snapshot export, compress and check --- src/lib_smart_rollup_node/snapshot_utils.ml | 30 +++++++++++++++++++++ src/lib_smart_rollup_node/snapshots.ml | 20 +++++++++++--- 2 files changed, 46 insertions(+), 4 deletions(-) diff --git a/src/lib_smart_rollup_node/snapshot_utils.ml b/src/lib_smart_rollup_node/snapshot_utils.ml index 9f3bbae3e298..c6d884ac5567 100644 --- a/src/lib_smart_rollup_node/snapshot_utils.ml +++ b/src/lib_smart_rollup_node/snapshot_utils.ml @@ -151,12 +151,31 @@ let list_files dir ~include_file f = let dir_handle = Unix.opendir dir in list_files_in_dir Stream.sempty (dir, "", dir_handle) +let total_bytes_to_export dir ~include_file = + let file_stream = + list_files dir ~include_file @@ fun ~full_path ~relative_path:_ -> + let {Unix.st_size; _} = Unix.lstat full_path in + st_size + in + let total = ref 0 in + Stream.iter (fun size -> total := !total + size) file_stream ; + !total + let create (module Reader : READER) (module Writer : WRITER) metadata ~dir ~include_file ~dest = let module Archive_writer = Tar.Make (struct include Reader include Writer end) in + let total = total_bytes_to_export dir ~include_file in + let progress_bar = + Progress_bar.progress_bar + ~counter:`Bytes + ~message:"Exporting snapshot " + ~color:(Terminal.Color.rgb 3 132 252) + total + in + Progress_bar.with_reporter progress_bar @@ fun count_progress -> let write_file file (out_chan : Writer.out_channel) = let in_chan = Reader.open_in file in try @@ -165,6 +184,7 @@ let create (module Reader : READER) (module Writer : WRITER) metadata ~dir let rec copy () = let read_bytes = Reader.input in_chan buf 0 buffer_size in Writer.output out_chan buf 0 read_bytes ; + count_progress read_bytes ; if read_bytes > 0 then copy () in copy () ; @@ -246,6 +266,15 @@ let extract (module Reader : READER) (module Writer : WRITER) metadata_check raise e let compress ~snapshot_file = + let Unix.{st_size = total; _} = Unix.stat snapshot_file in + let progress_bar = + Progress_bar.progress_bar + ~counter:`Bytes + ~message:"Compressing snapshot" + ~color:(Terminal.Color.rgb 3 198 252) + total + in + Progress_bar.with_reporter progress_bar @@ fun count_progress -> let snapshot_file_gz = Filename.chop_suffix snapshot_file ".uncompressed" in let in_chan = open_in snapshot_file in let out_chan = Gzip.open_out snapshot_file_gz in @@ -255,6 +284,7 @@ let compress ~snapshot_file = let rec copy () = let read_bytes = input in_chan buf 0 buffer_size in Gzip.output out_chan buf 0 read_bytes ; + count_progress read_bytes ; if read_bytes > 0 then copy () in copy () ; diff --git a/src/lib_smart_rollup_node/snapshots.ml b/src/lib_smart_rollup_node/snapshots.ml index ce7fd9721292..b8d69a66371c 100644 --- a/src/lib_smart_rollup_node/snapshots.ml +++ b/src/lib_smart_rollup_node/snapshots.ml @@ -94,10 +94,21 @@ let check_some hash what = function | None -> error_with "Could not read %s at %a after export." what Block_hash.pp hash -let check_l2_chain ~data_dir (store : _ Store.t) context +let check_l2_chain ~message ~data_dir (store : _ Store.t) context (head : Sc_rollup_block.t) = let open Lwt_result_syntax in let* first_available_level = first_available_level ~data_dir store in + let blocks_to_check = + Int32.sub head.header.level first_available_level |> Int32.to_int |> succ + in + let progress_bar = + Progress_bar.progress_bar + ~counter:`Int + ~message + ~color:(Terminal.Color.rgb 3 252 132) + blocks_to_check + in + Progress_bar.Lwt.with_reporter progress_bar @@ fun count_progress -> let rec check_block hash = let* b = Store.L2_blocks.read store.l2_blocks hash in let*? _b, header = check_some hash "L2 block" b in @@ -118,12 +129,13 @@ let check_l2_chain ~data_dir (store : _ Store.t) context (* Ensure head context is available. *) let*! head_ctxt = Context.checkout context header.context in let*? _head_ctxt = check_some hash "context" head_ctxt in + let*! () = count_progress 1 in if header.level <= first_available_level then return_unit else check_block header.predecessor in check_block head.header.block_hash -let post_import_checks ~dest = +let post_import_checks ~message ~dest = let open Lwt_result_syntax in let store_dir = Configuration.default_storage_dir dest in let context_dir = Configuration.default_context_dir dest in @@ -138,7 +150,7 @@ let post_import_checks ~dest = store_dir in let* head = check_head store context in - let* () = check_l2_chain ~data_dir:dest store context head in + let* () = check_l2_chain ~message ~data_dir:dest store context head in let*! () = Context.close context in let* () = Store.close store in return_unit @@ -146,7 +158,7 @@ let post_import_checks ~dest = let post_export_checks ~snapshot_file = Lwt_utils_unix.with_tempdir "snapshot_checks_" @@ fun dest -> extract gzip_reader stdlib_writer (fun _ -> ()) ~snapshot_file ~dest ; - post_import_checks ~dest + post_import_checks ~message:"Checking snapshot " ~dest let operator_local_file_regexp = Re.Str.regexp "^storage/\\(commitments_published_at_level.*\\|lpc$\\)" -- GitLab From f3783028e79ff125dde71b5d0589b6e736599ef9 Mon Sep 17 00:00:00 2001 From: Alain Mebsout Date: Fri, 1 Dec 2023 15:31:50 +0100 Subject: [PATCH 9/9] Doc: Changelog for snapshot export --- CHANGES.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index b88d4ebe1411..cdc84d1873c7 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -153,6 +153,8 @@ Smart Rollup node history, no GC happens. In ``full`` the rollup node retains data for possible refutations. (MRs :gl:`!10475`, :gl:`!10695`) +- Snapshot export with integrity checks. (MR :gl:`!10704`) + Smart Rollup client ------------------- -- GitLab