From a7e557931dfdcde6aa4294663d0d22af7f337b00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20El=20Siba=C3=AFe?= Date: Tue, 7 Feb 2023 11:23:07 +0100 Subject: [PATCH 1/2] Log sinks: add an option to create log directory if necessary --- CHANGES.rst | 3 +++ docs/user/logging.rst | 8 ++++++-- src/lib_stdlib_unix/file_descriptor_sink.ml | 6 ++++++ src/lib_stdlib_unix/file_descriptor_sink.mli | 2 ++ src/lib_stdlib_unix/lwt_utils_unix.mli | 2 ++ 5 files changed, 19 insertions(+), 2 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 049a1870ca77..4145a6db7f4f 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -53,6 +53,9 @@ Node - Replaced some "precheck" occurrences with "validate" in event and error identifiers and messages. (MR :gl: `!7513`) +- Added an option ``create-dirs`` to file-descriptor sinks to allow + the node to create the log directory and its parents if they don't exist. + Client ------ diff --git a/docs/user/logging.rst b/docs/user/logging.rst index 913c15b7228f..261cc42fd99a 100644 --- a/docs/user/logging.rst +++ b/docs/user/logging.rst @@ -112,6 +112,9 @@ Options available only for the ``file-descriptor-path://`` case: added as a suffix to the filename, prefixed by a dash. We recomend not to put your tezos logs in ``/var/log`` if you use this option, as you system would use ``logrotate`` automatically. +- ``create-dirs=`` when ``true`` allows to create the directory where + the log files are stored and all its parents recursively if they don't + exist. Examples: @@ -128,10 +131,11 @@ Examples: - ``file-descriptor-path:///the/path/to/write.log?section-prefix=rpc:debug§ion-prefix=validator:debug§ion-prefix=:none"`` → Write only sections validator and rpc at debug level but exclude all other sections from the stream. -- ``"file-descriptor-path:///tmp/node.log?daily-logs=5§ion-prefix=:info"`` +- ``"file-descriptor-path:///tmp/node-logs/node.log?daily-logs=5&create-dirs=true§ion-prefix=:info"`` sets up daily log files with a history of up to 5 days and verbosity level ``info`` for all logs. Files will be named ``node-19700101.log`` in an - example of a file produced in 1970, January, the 1st. + example of a file produced in 1970, January, the 1st. The log directory + ``node-logs`` will be automatically created if it doesn't exist. The format of the events is (usually minified): diff --git a/src/lib_stdlib_unix/file_descriptor_sink.ml b/src/lib_stdlib_unix/file_descriptor_sink.ml index 9b5f8cb89e51..8d8621323517 100644 --- a/src/lib_stdlib_unix/file_descriptor_sink.ml +++ b/src/lib_stdlib_unix/file_descriptor_sink.ml @@ -227,6 +227,12 @@ end) : Internal_event.SINK with type t = t = struct | "" | "/" -> fail_parsing uri "Missing path configuration." | path -> return path in + let allow_create_dir = flag "create-dirs" in + let*! () = + if allow_create_dir then + Lwt_utils_unix.create_dir (Filename.dirname path) + else Lwt.return_unit + in let open Lwt_result_syntax in let time_ext, rotation = match rotate with diff --git a/src/lib_stdlib_unix/file_descriptor_sink.mli b/src/lib_stdlib_unix/file_descriptor_sink.mli index a60407356985..11a7e34c48d6 100644 --- a/src/lib_stdlib_unix/file_descriptor_sink.mli +++ b/src/lib_stdlib_unix/file_descriptor_sink.mli @@ -59,6 +59,8 @@ last N days where N is the given parameter. It creates a file for each day and and adds the day of the year with format ["yyyymmdd"] to the path provided. + - ["create-dirs=true"] allows to create the directory where the log + files are stored and all its parents recursively if they don't exist. Examples: diff --git a/src/lib_stdlib_unix/lwt_utils_unix.mli b/src/lib_stdlib_unix/lwt_utils_unix.mli index caa5d0bf8e0e..a86bfce1bb28 100644 --- a/src/lib_stdlib_unix/lwt_utils_unix.mli +++ b/src/lib_stdlib_unix/lwt_utils_unix.mli @@ -81,6 +81,8 @@ val is_directory : string -> bool Lwt.t val remove_dir : string -> unit Lwt.t +(** [create_dir ?perm dir] creates the directory at the path [dir] and + its parents recursively if they doesn't exist *) val create_dir : ?perm:int -> string -> unit Lwt.t (** [copy_dir ?perm src dst] copies the content of directory [src] in -- GitLab From ffd92f9f20d0abc87bc0feb6e888719ecd5b1d09 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20El=20Siba=C3=AFe?= Date: Mon, 6 Feb 2023 18:14:16 +0100 Subject: [PATCH 2/2] Node/logs: enables daily logs on disk by default --- CHANGES.rst | 3 +++ docs/user/logging.rst | 10 +++++++--- src/bin_node/node_run_command.ml | 7 ++++++- src/bin_snoop/main_snoop.ml | 2 +- src/lib_base/internal_event_config.ml | 8 +++++++- src/lib_base/internal_event_config.mli | 17 +++++++++++++++-- src/lib_base/unix/internal_event_unix.ml | 2 +- src/lib_node_config/config_file.ml | 24 ++++++++++++++++++++++-- src/lib_node_config/config_file.mli | 4 ++++ 9 files changed, 66 insertions(+), 11 deletions(-) diff --git a/CHANGES.rst b/CHANGES.rst index 4145a6db7f4f..3c3e3fa96a75 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -56,6 +56,9 @@ Node - Added an option ``create-dirs`` to file-descriptor sinks to allow the node to create the log directory and its parents if they don't exist. +- Added a default node configuration that enables disk logs as a + file-descriptor-sink in the data directory of the node with a 7 days rotation. + Client ------ diff --git a/docs/user/logging.rst b/docs/user/logging.rst index 261cc42fd99a..568f0e1ac5f8 100644 --- a/docs/user/logging.rst +++ b/docs/user/logging.rst @@ -1,7 +1,7 @@ Logging ======= -Logging features in Tezos allow to monitor its execution and be informed in real +Logging features in Octez allow to monitor its execution and be informed in real time about *events* of interest, such as errors, completion of certain steps, etc. This is why various software components emit *events* throughout the codebase (see :doc:`../developer/event_logging_framework`), the logging @@ -202,8 +202,12 @@ where ```` is either ``no-section`` or Global Defaults --------------- -By default, only the ``lwt-log://`` sinks are activated and configured to -output events of level at least ``Notice``. +- ``lwt-log://`` sinks are activated by default and configured to + output events of level at least ``Notice``. Their goal is the stdout logging. + +- A file-descriptor-sink is activated to store logs from last seven days with + an ``Info`` level. The path is ``/daily-logs/``. + JSON Configuration Format ------------------------- diff --git a/src/bin_node/node_run_command.ml b/src/bin_node/node_run_command.ml index f59546905ce3..143c554ac59c 100644 --- a/src/bin_node/node_run_command.ml +++ b/src/bin_node/node_run_command.ml @@ -510,10 +510,15 @@ let run ?verbosity ?sandbox ?target ?(cli_warnings = []) | None -> config.log | Some default_level -> {config.log with default_level} in + let internal_events = + if Internal_event_config.(is_empty config.internal_events) then + Config_file.make_default_internal_events ~data_dir:config.data_dir + else config.internal_events + in let*! () = Tezos_base_unix.Internal_event_unix.init ~lwt_log_sink:log_cfg - ~configuration:config.internal_events + ~configuration:internal_events () in let*! () = diff --git a/src/bin_snoop/main_snoop.ml b/src/bin_snoop/main_snoop.ml index b5b20d841cad..b3c58c465664 100644 --- a/src/bin_snoop/main_snoop.ml +++ b/src/bin_snoop/main_snoop.ml @@ -521,7 +521,7 @@ let () = @@ Tezos_base_unix.Internal_event_unix.( init ~lwt_log_sink:Lwt_log_sink_unix.default_cfg - ~configuration:Configuration.default) + ~configuration:Configuration.lwt_log) () let () = diff --git a/src/lib_base/internal_event_config.ml b/src/lib_base/internal_event_config.ml index 3101560892b6..2659f9fe2227 100644 --- a/src/lib_base/internal_event_config.ml +++ b/src/lib_base/internal_event_config.ml @@ -27,9 +27,15 @@ open Error_monad type t = {active_sinks : Uri.t list} -let default = +let lwt_log = {active_sinks = [Uri.make ~scheme:Internal_event.Lwt_log_sink.uri_scheme ()]} +let empty = {active_sinks = []} + +let is_empty {active_sinks} = active_sinks = [] + +let make_custom active_sinks = {active_sinks} + let encoding = let open Data_encoding in let object_1 key_name = diff --git a/src/lib_base/internal_event_config.mli b/src/lib_base/internal_event_config.mli index 273583fd7344..c51d5cd3f6d4 100644 --- a/src/lib_base/internal_event_config.mli +++ b/src/lib_base/internal_event_config.mli @@ -29,12 +29,25 @@ internal-events framework. It allows one to activate registered event sinks. *) +(** FIXME: https://gitlab.com/tezos/tezos/-/issues/4850 + Config override default values, but the env TEZOS_EVENTS_CONFIG + does not. Is that what we want ? *) + open Error_monad type t -(** The default configuration is empty (it doesn't activate any sink). *) -val default : t +(** The empty configuration. It doesn't activate any sink. *) +val empty : t + +(** The configuration with only an lwt-log sink. *) +val lwt_log : t + +(** Check if the configuration is empty. *) +val is_empty : t -> bool + +(** Allows to make custom list of uris. *) +val make_custom : Uri.t list -> t (** The serialization format. *) val encoding : t Data_encoding.t diff --git a/src/lib_base/unix/internal_event_unix.ml b/src/lib_base/unix/internal_event_unix.ml index 9c5317f0e16c..e6a99da3e1a8 100644 --- a/src/lib_base/unix/internal_event_unix.ml +++ b/src/lib_base/unix/internal_event_unix.ml @@ -42,7 +42,7 @@ end run with [Singleprocess]. *) let env_var_name = "TEZOS_EVENTS_CONFIG" -let init ?lwt_log_sink ?(configuration = Configuration.default) () = +let init ?lwt_log_sink ?(configuration = Configuration.lwt_log) () = let _ = (* This is just here to force the linking (and hence initialization) of all these modules: *) diff --git a/src/lib_node_config/config_file.ml b/src/lib_node_config/config_file.ml index f5588a3a5052..77f82eb80fc8 100644 --- a/src/lib_node_config/config_file.ml +++ b/src/lib_node_config/config_file.ml @@ -374,6 +374,26 @@ let sugared_blockchain_network_encoding : blockchain_network Data_encoding.t = (fun x -> x); ]) +let make_default_internal_events ~data_dir = + (* By default the node has two logs output: + - on stdout using Lwt_log using the configured verbosity + - on disk, with a 7 days rotation with an info verbosity level *) + Internal_event_config.make_custom + [ + Uri.make ~scheme:Internal_event.Lwt_log_sink.uri_scheme (); + Uri.make + ~scheme:"file-descriptor-path" + ~path:(data_dir // "daily-logs" // "daily.log") + ~query: + [ + ("create-dirs", ["true"]); + ("daily-logs", ["7"]); + ("section-prefix", [":info"]); + ("format", ["pp"]); + ] + (); + ] + type t = { data_dir : string; disable_config_validation : bool; @@ -443,7 +463,7 @@ let default_config = p2p = default_p2p; rpc = default_rpc; log = Lwt_log_sink_unix.default_cfg; - internal_events = Internal_event_config.default; + internal_events = Internal_event_config.empty; shell = Shell_limits.default_limits; blockchain_network = blockchain_network_mainnet; disable_config_validation = default_disable_config_validation; @@ -732,7 +752,7 @@ let encoding = "internal-events" ~description:"Configuration of the structured logging framework" Internal_event_config.encoding - Internal_event_config.default) + Internal_event_config.empty) (dft "shell" ~description:"Configuration of network parameters" diff --git a/src/lib_node_config/config_file.mli b/src/lib_node_config/config_file.mli index e34302d30ba8..e5b1fcc483b9 100644 --- a/src/lib_node_config/config_file.mli +++ b/src/lib_node_config/config_file.mli @@ -83,6 +83,10 @@ and rpc = { and tls = {cert : string; key : string} +(** Creates the default internal event configuration using [data_dir] *) +val make_default_internal_events : + data_dir:string -> Tezos_base.Internal_event_config.t + val data_dir_env_name : string val default_data_dir : string -- GitLab