From afcd5faaedf6206b4ad4cecfbf95228f56b80bd4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Thu, 26 Sep 2024 01:45:54 +0200 Subject: [PATCH 1/3] KVS: Add a view counting the number of "opened" files --- src/lib_stdlib_unix/key_value_store.ml | 12 ++++++++++++ src/lib_stdlib_unix/key_value_store.mli | 8 ++++++++ 2 files changed, 20 insertions(+) diff --git a/src/lib_stdlib_unix/key_value_store.ml b/src/lib_stdlib_unix/key_value_store.ml index 5376e445a17f..7d80958710bf 100644 --- a/src/lib_stdlib_unix/key_value_store.ml +++ b/src/lib_stdlib_unix/key_value_store.ml @@ -227,6 +227,10 @@ module Files : sig val count_values : 'value t -> ('key, 'value) layout -> int tzresult Lwt.t val remove : 'value t -> ('key, 'value) layout -> unit tzresult Lwt.t + + module View : sig + val opened_files : 'value t -> int + end end = struct module LRU = Ringo.LRU_Collection @@ -346,6 +350,10 @@ end = struct The store ensures that actions performed on a given file are done sequentially. *) + module View = struct + let opened_files {files; _} = Table.length files + end + let init ~lru_size = (* FIXME https://gitlab.com/tezos/tezos/-/issues/6774 @@ -1066,6 +1074,10 @@ let remove_file {files; root_dir; _} file_layout file = let layout = file_layout ~root_dir file in Files.remove files layout +module View = struct + let opened_files {files; _} = Files.View.opened_files files +end + module Internal_for_tests = struct let init ?(lockfile_prefix = "internal_for_tests") ~lru_size ~root_dir () = let open Lwt_result_syntax in diff --git a/src/lib_stdlib_unix/key_value_store.mli b/src/lib_stdlib_unix/key_value_store.mli index 4c88622b63dc..60b8bf3c5c59 100644 --- a/src/lib_stdlib_unix/key_value_store.mli +++ b/src/lib_stdlib_unix/key_value_store.mli @@ -208,6 +208,14 @@ val count_values : 'file -> int tzresult Lwt.t +module View : sig + (** Returns the number of files current opened by the key value + store. Do note this number is an upper bound on the number of + file descriptors opened. + *) + val opened_files : ('file, 'key, 'value) t -> int +end + module Internal_for_tests : sig (** Same as {!init} above, except that the user can specify a prefix for the lock file (default is lockfile_prefix = "internal_for_tests") to avoid issues -- GitLab From 7977804dfc1fdb3e9e03ac5b294bfba8aec66c71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Thu, 26 Sep 2024 01:46:06 +0200 Subject: [PATCH 2/3] KVS/Test: Ensure the number of opened files is valid --- .../test/test_key_value_store_fuzzy.ml | 67 +++++++++---------- 1 file changed, 32 insertions(+), 35 deletions(-) diff --git a/src/lib_stdlib_unix/test/test_key_value_store_fuzzy.ml b/src/lib_stdlib_unix/test/test_key_value_store_fuzzy.ml index ed193cba24d6..56c56d79aff0 100644 --- a/src/lib_stdlib_unix/test/test_key_value_store_fuzzy.ml +++ b/src/lib_stdlib_unix/test/test_key_value_store_fuzzy.ml @@ -96,6 +96,10 @@ module type S = sig ('file, 'key, 'value) Key_value_store.file_layout -> 'file -> int tzresult Lwt.t + + module View : sig + val opened_files : ('file, 'key, 'value) t -> int + end end let value_size = 1 @@ -142,6 +146,12 @@ module R : S = struct (fun (file', _) _ count -> if file = file' then count + 1 else count) t 0 + + module View = struct + let opened_files table = + table |> Stdlib.Hashtbl.to_seq_keys |> Seq.map fst |> List.of_seq + |> List.sort_uniq compare |> List.length + end end module Helpers = struct @@ -374,46 +384,26 @@ module Helpers = struct let pp_scenario fmt (action, next_actions) = let rec pp shift action fmt next_actions = - let shift_str = "|| " in - if shift then Format.fprintf fmt "%s " shift_str ; match next_actions with | [] -> Format.fprintf fmt "%a@." pp_action action ; if shift then Format.fprintf fmt "Wait" | (Parallel, next_action) :: actions -> - if shift then - Format.fprintf - fmt - "%a@.%a" - pp_action - action - (pp true next_action) - actions - else - Format.fprintf - fmt - "%a@.Wait@.%a" - pp_action - action - (pp true next_action) - actions + Format.fprintf + fmt + "P %a@.%a" + pp_action + action + (pp true next_action) + actions | (Sequential, next_action) :: actions -> - if shift then - Format.fprintf - fmt - "Wait@.%a@.%a@." - pp_action - action - (pp false next_action) - actions - else - Format.fprintf - fmt - "%a@.%a@." - pp_action - action - (pp false next_action) - actions + Format.fprintf + fmt + "S %a@.%a@." + pp_action + action + (pp false next_action) + actions in Format.fprintf fmt "%a" (pp false action) next_actions end @@ -578,7 +568,14 @@ let run_scenario (function Ok () -> return_unit | Error err -> fail err) promises_running_seq in - run_actions action next_actions Seq_s.empty + if L.View.opened_files left = min lru_size (R.View.opened_files right) + then run_actions action next_actions Seq_s.empty + else + failwith + "Expected size of files table to be %d. Got %d (lru size is: %d)." + (R.View.opened_files right) + (L.View.opened_files left) + lru_size | (Parallel, action) :: next_actions -> (* We do not wait for promises to end and append them to the list of promises on-going. *) -- GitLab From 515da68ad7cbdb84502fe01b90b83d545b27171a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Fran=C3=A7ois=20Thir=C3=A9?= Date: Thu, 26 Sep 2024 01:44:54 +0200 Subject: [PATCH 3/3] KVS: Clean up table after removing files from the KVS --- src/lib_stdlib_unix/key_value_store.ml | 5 ++++- src/lib_stdlib_unix/key_value_store.mli | 2 +- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib_stdlib_unix/key_value_store.ml b/src/lib_stdlib_unix/key_value_store.ml index 7d80958710bf..227159a6e6bf 100644 --- a/src/lib_stdlib_unix/key_value_store.ml +++ b/src/lib_stdlib_unix/key_value_store.ml @@ -862,7 +862,10 @@ end = struct if !closed then Lwt.return (Error (Error_monad.TzTrace.make (Closed {action = "remove"}))) else - let on_file_closed ~on_file_opened:_ = may_remove_file layout.filepath in + let on_file_closed ~on_file_opened:_ = + let+ () = may_remove_file layout.filepath in + Table.remove files layout.filepath + in let p = Action.remove_file ~on_file_closed diff --git a/src/lib_stdlib_unix/key_value_store.mli b/src/lib_stdlib_unix/key_value_store.mli index 60b8bf3c5c59..e9fc0e46b379 100644 --- a/src/lib_stdlib_unix/key_value_store.mli +++ b/src/lib_stdlib_unix/key_value_store.mli @@ -209,7 +209,7 @@ val count_values : int tzresult Lwt.t module View : sig - (** Returns the number of files current opened by the key value + (** Returns the number of files currently opened by the key value store. Do note this number is an upper bound on the number of file descriptors opened. *) -- GitLab