From 1c488778423a60e645722a1cfb77d84555c24808 Mon Sep 17 00:00:00 2001 From: Guillaume Bau Date: Mon, 10 Mar 2025 18:03:48 +0100 Subject: [PATCH 1/4] Tezt/Cloud: adds a logrotate module --- tezt/lib_cloud/dockerfiles/dal.Dockerfile | 2 + tezt/lib_cloud/logrotate.ml | 49 +++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tezt/lib_cloud/logrotate.ml diff --git a/tezt/lib_cloud/dockerfiles/dal.Dockerfile b/tezt/lib_cloud/dockerfiles/dal.Dockerfile index a23a27c04b7a..8e99125669e0 100644 --- a/tezt/lib_cloud/dockerfiles/dal.Dockerfile +++ b/tezt/lib_cloud/dockerfiles/dal.Dockerfile @@ -21,6 +21,8 @@ RUN apt-get update && apt-get install -y \ iproute2 \ # Can be used to monitor process individually prometheus-process-exporter \ + # rotation for logs + logrotate \ # emacs can be useful for debugging emacs \ # wget can be used to import snapshots diff --git a/tezt/lib_cloud/logrotate.ml b/tezt/lib_cloud/logrotate.ml new file mode 100644 index 000000000000..731e2e89ed56 --- /dev/null +++ b/tezt/lib_cloud/logrotate.ml @@ -0,0 +1,49 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2025 Nomadic Labs *) +(* *) +(*****************************************************************************) + +let config name = Format.asprintf "/etc/logrotate.d/%s" name + +let write_config ~name ~target_file agent = + (* Write file locally *) + let source = Temp.file (Format.asprintf "%s.logrotate" (Agent.name agent)) in + (* Choose a reasonable 300 rotations of 200MB max file + Files after 300 rotations are deleted + The rotate is triggered daily or if they are greater than 200MB *) + Base.with_open_out source (fun oc -> + output_string + oc + (Format.asprintf + "%s {@\n%s" + target_file + {| +# Rotates logs every day + daily +# Rotates the log when it reaches 200 megabytes, even if not time for rotation + maxsize 200M +# Skips rotation if the log file is empty + notifempty +# Doesn't produce an error if the log file is missing + missingok +# Compresses rotated logs using gzip by default + compress +# Copies the log file then truncates the original instead of moving it + copytruncate +# Runs rotation commands with root user and group permissions + su root root +# Maximum number of rotations before removing the oldests + rotate 300 +} +|})) ; + (* Upload the generated config in the destination /etc/logrotate.d/$name *) + let destination = config name in + let* _ = Agent.copy agent ~source ~destination in + Lwt.return_unit + +let run ~name agent = + let name = config name in + (* Run logrotate as root in the docker container *) + Agent.docker_run_command agent "logrotate" [name] |> Process.check -- GitLab From f56f79f96d15a08d82fd73278843505e78056abf Mon Sep 17 00:00:00 2001 From: Guillaume Bau Date: Mon, 10 Mar 2025 18:04:55 +0100 Subject: [PATCH 2/4] Tezt/Cloud: enable chronos task for logrotating tezt logs --- tezt/lib_cloud/cloud.ml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/tezt/lib_cloud/cloud.ml b/tezt/lib_cloud/cloud.ml index 03575d4c8590..861b1a193522 100644 --- a/tezt/lib_cloud/cloud.ml +++ b/tezt/lib_cloud/cloud.ml @@ -169,6 +169,42 @@ let orchestrator ?(alerts = []) ?(tasks = []) deployement f = Lwt.return_some prometheus else Lwt.return_none in + (* FIXME: env/cli instead of true *) + let logrotate = true in + (* Logrotate: write a configuration file inside each container *) + let* logrotate = + match (logrotate, Tezt.Cli.Logs.file) with + (* If no logfile, do not enable logrotate *) + | _, None -> Lwt.return false + (* If logfile and logrotate, configure each agent *) + | true, Some target_file -> + let* () = + Lwt_list.iter_s + (fun agent -> + Logrotate.write_config ~name:"tezt-cloud" ~target_file agent) + agents + in + Lwt.return true + (* If no logrorate, return false *) + | false, _ -> Lwt.return_false + in + let tasks = + if logrotate then + let name = "logrotate" in + (* chronos: triggers logrotate every 4 hours. + if each 4 hours, the criteria to trigger a rotation is meet, they + will be rotated. + note that rotation involves a in important IO stress for the machine *) + let tm = "0 1-23/4 * * *" in + let action () = + Lwt_list.iter_s + (fun agent -> Logrotate.run ~name:"tezt-cloud" agent) + agents + in + let task = Chronos.task ~name ~tm ~action in + task :: tasks + else tasks + in let chronos = if List.is_empty tasks then None else -- GitLab From 7a51e31e77181e9a01f07ab56e5ecbf18298087b Mon Sep 17 00:00:00 2001 From: Guillaume Bau Date: Tue, 11 Mar 2025 16:17:02 +0100 Subject: [PATCH 3/4] Tezt/Cloud/Logrotate: adds some logs --- tezt/lib_cloud/logrotate.ml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tezt/lib_cloud/logrotate.ml b/tezt/lib_cloud/logrotate.ml index 731e2e89ed56..bd8e2e4cde93 100644 --- a/tezt/lib_cloud/logrotate.ml +++ b/tezt/lib_cloud/logrotate.ml @@ -38,12 +38,19 @@ let write_config ~name ~target_file agent = rotate 300 } |})) ; + Log.info "Teztcloud.Logrotate: written configuration for %s" target_file ; (* Upload the generated config in the destination /etc/logrotate.d/$name *) let destination = config name in + Log.info + "Teztcloud.Logrotate: uploading %s to %s:%s" + target_file + (Agent.name agent) + destination ; let* _ = Agent.copy agent ~source ~destination in Lwt.return_unit let run ~name agent = let name = config name in (* Run logrotate as root in the docker container *) + Log.info "Teztcloud.Logrotate: executing task %s" name ; Agent.docker_run_command agent "logrotate" [name] |> Process.check -- GitLab From 3230b8caeb28fd266568501244b750c02a0d2a91 Mon Sep 17 00:00:00 2001 From: Guillaume Bau Date: Tue, 11 Mar 2025 16:20:00 +0100 Subject: [PATCH 4/4] Tezt/Cloud/Logrotate: adds cli option --log-rotation, set to 0 to disable Logrotation is now enabled by default if tezt was invoked with `--log-file`. If `--log-rotation` is specified to 0, logrotate will be completely disabled. If different to 0, it will represent the number of old rotated log files to keep before deleting the oldest ones. --- tezt/lib_cloud/cli.ml | 10 ++++++++++ tezt/lib_cloud/cli.mli | 5 +++++ tezt/lib_cloud/cloud.ml | 21 +++++++++++++-------- tezt/lib_cloud/env.ml | 2 ++ tezt/lib_cloud/env.mli | 3 +++ tezt/lib_cloud/logrotate.ml | 11 +++++------ tezt/lib_cloud/logrotate.mli | 19 +++++++++++++++++++ 7 files changed, 57 insertions(+), 14 deletions(-) create mode 100644 tezt/lib_cloud/logrotate.mli diff --git a/tezt/lib_cloud/cli.ml b/tezt/lib_cloud/cli.ml index dc9fd905da7b..cac2031b1d77 100644 --- a/tezt/lib_cloud/cli.ml +++ b/tezt/lib_cloud/cli.ml @@ -370,3 +370,13 @@ let binaries_path = "Where to find binaries in the docker image by default (default is: \ '/tmp/tezt-runners')" Types.Agent_configuration.default_gcp_binaries_path + +let log_rotation = + Clap.default_int + ~section + ~long:"log-rotation" + ~description: + "Maximum number of log rotations before removing older log files. \ + Default is 300 if a log-file is specified.\n\ + \ Set to 0 to completely disable log-rotation" + 300 diff --git a/tezt/lib_cloud/cli.mli b/tezt/lib_cloud/cli.mli index 178f8b08b251..cf80d28babd1 100644 --- a/tezt/lib_cloud/cli.mli +++ b/tezt/lib_cloud/cli.mli @@ -138,3 +138,8 @@ val faketime : string option (** Where to find binaries path by default in the docker image. *) val binaries_path : string + +(** How many log rotation until we remove older logs + Defaults to 300 + Use 0 to disable log-rotation *) +val log_rotation : int diff --git a/tezt/lib_cloud/cloud.ml b/tezt/lib_cloud/cloud.ml index 861b1a193522..897af9551835 100644 --- a/tezt/lib_cloud/cloud.ml +++ b/tezt/lib_cloud/cloud.ml @@ -169,33 +169,38 @@ let orchestrator ?(alerts = []) ?(tasks = []) deployement f = Lwt.return_some prometheus else Lwt.return_none in - (* FIXME: env/cli instead of true *) - let logrotate = true in - (* Logrotate: write a configuration file inside each container *) + (* Enable logrotate if --log_rotation was not set to 0... *) + let logrotate = Env.log_rotation <> 0 in let* logrotate = + (* ... and if log-file was specified *) match (logrotate, Tezt.Cli.Logs.file) with (* If no logfile, do not enable logrotate *) | _, None -> Lwt.return false (* If logfile and logrotate, configure each agent *) | true, Some target_file -> let* () = + (* Logrotate: write a configuration file inside each container *) Lwt_list.iter_s (fun agent -> - Logrotate.write_config ~name:"tezt-cloud" ~target_file agent) + Logrotate.write_config + ~name:"tezt-cloud" + ~target_file + ~max_rotations:Env.log_rotation + agent) agents in Lwt.return true - (* If no logrorate, return false *) + (* If no logrotate, return false *) | false, _ -> Lwt.return_false in let tasks = if logrotate then let name = "logrotate" in (* chronos: triggers logrotate every 4 hours. - if each 4 hours, the criteria to trigger a rotation is meet, they + if each 4 hours, the criteria to trigger a rotation is met, they will be rotated. - note that rotation involves a in important IO stress for the machine *) - let tm = "0 1-23/4 * * *" in + Note that rotation may involve an important IO stress for the machine *) + let tm = "0 0-23/4 * * *" in let action () = Lwt_list.iter_s (fun agent -> Logrotate.run ~name:"tezt-cloud" agent) diff --git a/tezt/lib_cloud/env.ml b/tezt/lib_cloud/env.ml index 7b5a64f2f990..0f43ea86b1ae 100644 --- a/tezt/lib_cloud/env.ml +++ b/tezt/lib_cloud/env.ml @@ -92,6 +92,8 @@ let binaries_path = Cli.binaries_path let process_monitoring = Cli.process_monitoring +let log_rotation = Cli.log_rotation + let init () = if tezt_cloud = "" then Test.fail diff --git a/tezt/lib_cloud/env.mli b/tezt/lib_cloud/env.mli index 70d9334dbfd1..a507c48fe41c 100644 --- a/tezt/lib_cloud/env.mli +++ b/tezt/lib_cloud/env.mli @@ -130,6 +130,9 @@ val faketime : string option (** Equivalent to [Cli.binaries_path]. *) val binaries_path : string +(** Equivalent to [Cli.log_rotation] *) +val log_rotation : int + (** [init ()] initialises and deploys a Docker registry using Terraform, only when the [mode] is either [`Host] or [`Cloud]. *) val init : unit -> unit Lwt.t diff --git a/tezt/lib_cloud/logrotate.ml b/tezt/lib_cloud/logrotate.ml index bd8e2e4cde93..895cd37b7ae8 100644 --- a/tezt/lib_cloud/logrotate.ml +++ b/tezt/lib_cloud/logrotate.ml @@ -7,7 +7,7 @@ let config name = Format.asprintf "/etc/logrotate.d/%s" name -let write_config ~name ~target_file agent = +let write_config ~name ~target_file ~max_rotations agent = (* Write file locally *) let source = Temp.file (Format.asprintf "%s.logrotate" (Agent.name agent)) in (* Choose a reasonable 300 rotations of 200MB max file @@ -32,12 +32,11 @@ let write_config ~name ~target_file agent = compress # Copies the log file then truncates the original instead of moving it copytruncate -# Runs rotation commands with root user and group permissions +# Runs rotation commands with root user and group permissions daily su root root -# Maximum number of rotations before removing the oldests - rotate 300 -} -|})) ; +# Maximum number of rotations before removing the oldests. +|} + ^ Format.asprintf " rotate %d\n}\n" max_rotations)) ; Log.info "Teztcloud.Logrotate: written configuration for %s" target_file ; (* Upload the generated config in the destination /etc/logrotate.d/$name *) let destination = config name in diff --git a/tezt/lib_cloud/logrotate.mli b/tezt/lib_cloud/logrotate.mli new file mode 100644 index 000000000000..a4b95cff8dc5 --- /dev/null +++ b/tezt/lib_cloud/logrotate.mli @@ -0,0 +1,19 @@ +(*****************************************************************************) +(* *) +(* SPDX-License-Identifier: MIT *) +(* SPDX-FileCopyrightText: 2025 Nomadic Labs *) +(* *) +(*****************************************************************************) + +(** [write_config name target_file max_rotations agent] will write a logrotate + configuration file for the rotation of [target_file], will a maximum rotation + of [max_rotations] on the [agent]. *) +val write_config : + name:string -> + target_file:string -> + max_rotations:int -> + Agent.t -> + unit Lwt.t + +(** [run name agent] will run the logrotate [name] task on the [agent] *) +val run : name:string -> Agent.t -> unit Lwt.t -- GitLab