From ed4bea78a26aae3e157c922abaef5805525d2e81 Mon Sep 17 00:00:00 2001 From: Marius Bobin Date: Thu, 24 Apr 2025 17:46:43 +0300 Subject: [PATCH 1/2] Make pipeline cleanup limits configurable Add an applicaiton setting to configure the upper limit for the auto pipeline cleanup feature. Changelog: other --- app/helpers/application_settings_helper.rb | 1 + app/models/application_setting.rb | 12 +++++++++- .../application_setting_implementation.rb | 1 + app/models/project_ci_cd_setting.rb | 7 ++++-- .../application_setting_ci_cd_settings.json | 4 ++++ .../projects/settings/ci_cd/_form.html.haml | 2 +- doc/administration/instance_limits.md | 19 +++++++++++++++ doc/api/settings.md | 1 + locale/gitlab.pot | 4 ++-- spec/models/application_setting_spec.rb | 23 ++++++++++++++++++- spec/models/project_ci_cd_setting_spec.rb | 16 +++++++++++++ 11 files changed, 83 insertions(+), 7 deletions(-) diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index d9e748d7b27cc4..bb38799844aeab 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -267,6 +267,7 @@ def visible_attributes :auto_devops_domain, :autocomplete_users_limit, :autocomplete_users_unauthenticated_limit, + :ci_delete_pipelines_in_seconds_limit_human_readable, :ci_job_live_trace_enabled, :ci_partitions_size_limit, :concurrent_github_import_jobs_limit, diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 385860e5d040d2..fda2f11fbaeeea 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -523,10 +523,15 @@ def self.kroki_formats_attributes jsonb_accessor :ci_cd_settings, ci_job_live_trace_enabled: [:boolean, { default: false }], - ci_partitions_size_limit: [::Gitlab::Database::Type::JsonbInteger.new, { default: 100.gigabytes }] + ci_partitions_size_limit: [::Gitlab::Database::Type::JsonbInteger.new, { default: 100.gigabytes }], + ci_delete_pipelines_in_seconds_limit: [:integer, { default: ChronicDuration.parse('1 year') }] + + chronic_duration_attr :ci_delete_pipelines_in_seconds_limit_human_readable, :ci_delete_pipelines_in_seconds_limit validate :validate_object_storage_for_live_trace_configuration, if: -> { ci_job_live_trace_enabled? } validates :ci_partitions_size_limit, presence: true, numericality: { only_integer: true, greater_than: 0 } + validates :ci_delete_pipelines_in_seconds_limit, presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 1.day } validates :default_ci_config_path, format: { without: %r{(\.{2}|\A/)}, message: N_('cannot include leading slash or directory traversal.') }, @@ -1154,6 +1159,11 @@ def failed_login_attempts_unlock_period_in_minutes_column_exists? self.class.database.cached_column_exists?(:failed_login_attempts_unlock_period_in_minutes) end + def ci_delete_pipelines_in_seconds_limit_human_readable_long + value = ci_delete_pipelines_in_seconds_limit + ChronicDuration.output(value, format: :long) if value + end + private def parsed_grafana_url diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index cd71001f8ea1e4..ceed2a91eaca04 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -49,6 +49,7 @@ def defaults # rubocop:disable Metrics/AbcSize ci_job_live_trace_enabled: false, ci_max_total_yaml_size_bytes: 314572800, # max_yaml_size_bytes * ci_max_includes = 2.megabyte * 150 ci_partitions_size_limit: 100.gigabytes, + ci_delete_pipelines_in_seconds_limit_human_readable: '1 year', commit_email_hostname: default_commit_email_hostname, container_expiration_policies_enable_historic_entries: false, container_registry_features: [], diff --git a/app/models/project_ci_cd_setting.rb b/app/models/project_ci_cd_setting.rb index 726feae71a937a..10c3e9658d01da 100644 --- a/app/models/project_ci_cd_setting.rb +++ b/app/models/project_ci_cd_setting.rb @@ -42,8 +42,11 @@ class ProjectCiCdSetting < ApplicationRecord numericality: { only_integer: true, greater_than_or_equal_to: ChronicDuration.parse('1 day'), - less_than_or_equal_to: ChronicDuration.parse('1 year'), - message: N_('must be between 1 day and 1 year') + less_than_or_equal_to: ->(_) { ::Gitlab::CurrentSettings.ci_delete_pipelines_in_seconds_limit }, + message: ->(*) { + format(N_('must be between 1 day and %{limit}'), + limit: ::Gitlab::CurrentSettings.ci_delete_pipelines_in_seconds_limit_human_readable_long) + } } attribute :forward_deployment_enabled, default: true diff --git a/app/validators/json_schemas/application_setting_ci_cd_settings.json b/app/validators/json_schemas/application_setting_ci_cd_settings.json index 2eaf126ad9bf1c..6e7afa7e9c2f18 100644 --- a/app/validators/json_schemas/application_setting_ci_cd_settings.json +++ b/app/validators/json_schemas/application_setting_ci_cd_settings.json @@ -11,6 +11,10 @@ "ci_partitions_size_limit": { "type": "integer", "description": "Maximum table size limit for a CI partition used for creating new partitions." + }, + "ci_delete_pipelines_in_seconds_limit": { + "type": "integer", + "description": "Maximum value allowed for configuring pipeline retention." } } } diff --git a/app/views/projects/settings/ci_cd/_form.html.haml b/app/views/projects/settings/ci_cd/_form.html.haml index 1dd8188180a253..2208cf5ef5aebb 100644 --- a/app/views/projects/settings/ci_cd/_form.html.haml +++ b/app/views/projects/settings/ci_cd/_form.html.haml @@ -92,7 +92,7 @@ = form.label :delete_pipelines_in_human_readable, s_('CICD|Automatic pipeline cleanup'), class: 'label-bold' = form.text_field :delete_pipelines_in_human_readable, { class: 'form-control gl-form-input' } %p.form-text.gl-text-subtle - = html_escape(s_("CICD|Pipelines older than the configured time are deleted. Leave empty to never delete pipelines automatically. The default unit is in seconds, but you can use other units, for example %{code_open}15 days%{code_close}, %{code_open}1 month%{code_close}, %{code_open}1 year%{code_close}. Can be between 1 day to 1 year.")) % { code_open: ''.html_safe, code_close: ''.html_safe } + = html_escape(s_("CICD|Pipelines older than the configured time are deleted. Leave empty to never delete pipelines automatically. The default unit is in seconds, but you can use other units, for example %{code_open}15 days%{code_close}, %{code_open}1 month%{code_close}, %{code_open}1 year%{code_close}. Can be between 1 day to %{limit}.")) % { code_open: ''.html_safe, code_close: ''.html_safe, limit: ::Gitlab::CurrentSettings.ci_delete_pipelines_in_seconds_limit_human_readable_long } = link_to sprite_icon('question-o'), help_page_path('ci/pipelines/settings.md', anchor: 'automatic-pipeline-cleanup'), target: '_blank', rel: 'noopener noreferrer' = f.submit _('Save changes'), pajamas_button: true diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 047495da5dbf0a..cf2af50757ab7e 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -987,6 +987,25 @@ To change the limit, update `ci_partitions_size_limit` with the new value. For e ApplicationSetting.update(ci_partitions_size_limit: 20.gigabytes) ``` +### Maximum config value for automatic pipeline cleanup + +{{< history >}} + +- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/189131) in GitLab 18.0. + +{{< /history >}} + +Configures the upper limit for [CI/CD pipeline expiry time](../ci/pipelines/settings.md#automatic-pipeline-cleanup). +Defaults to 1 year. + +You can change this limit by using the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session). +To change the limit, update `ci_delete_pipelines_in_seconds_limit_human_readable` with the new value. +For example, to set it to 3 years: + +```ruby +ApplicationSetting.update(ci_delete_pipelines_in_seconds_limit_human_readable: '3 years') +``` + ## Instance monitoring and metrics ### Limit inbound incident management alerts diff --git a/doc/api/settings.md b/doc/api/settings.md index 523e7ac93162cb..dde2a78c726a59 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -465,6 +465,7 @@ to configure other related settings. These requirements are | `bulk_import_max_download_file_size` | integer | no | Maximum download file size when importing from source GitLab instances by direct transfer. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/384976) in GitLab 16.3. | | `can_create_group` | boolean | no | Indicates whether users can create top-level groups. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367754) in GitLab 15.5. Defaults to `true`. | | `check_namespace_plan` | boolean | no | Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. Premium and Ultimate only. | +| `ci_delete_pipelines_in_seconds_limit_human_readable` | string | no | Maximum value that is allowed for configuring pipeline retention. Defaults to `1 year`. | | `ci_job_live_trace_enabled` | boolean | no | Turns on incremental logging for job logs. When turned on, archived job logs are incrementally uploaded to object storage. Object storage must be configured. You can also configure this setting in the [**Admin** area](../administration/settings/continuous_integration.md#incremental-logging). | | `ci_max_total_yaml_size_bytes` | integer | no | The maximum amount of memory, in bytes, that can be allocated for the pipeline configuration, with all included YAML configuration files. | | `ci_max_includes` | integer | no | The [maximum number of includes](../administration/settings/continuous_integration.md#maximum-includes) per pipeline. Default is `150`. | diff --git a/locale/gitlab.pot b/locale/gitlab.pot index d63b82bd655385..27bf636c4b730e 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -11740,7 +11740,7 @@ msgstr "" msgid "CICD|Pipelines and jobs cannot be cancelled" msgstr "" -msgid "CICD|Pipelines older than the configured time are deleted. Leave empty to never delete pipelines automatically. The default unit is in seconds, but you can use other units, for example %{code_open}15 days%{code_close}, %{code_open}1 month%{code_close}, %{code_open}1 year%{code_close}. Can be between 1 day to 1 year." +msgid "CICD|Pipelines older than the configured time are deleted. Leave empty to never delete pipelines automatically. The default unit is in seconds, but you can use other units, for example %{code_open}15 days%{code_close}, %{code_open}1 month%{code_close}, %{code_open}1 year%{code_close}. Can be between 1 day to %{limit}." msgstr "" msgid "CICD|Pipelines triggered by composite identities, including AI agents, run automatically without review or manual interaction." @@ -72635,7 +72635,7 @@ msgstr "" msgid "must be before %{expiry_date}" msgstr "" -msgid "must be between 1 day and 1 year" +msgid "must be between 1 day and %{limit}" msgstr "" msgid "must be enabled." diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 799ff70640c99c..10bcf59f5d0bfe 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -96,7 +96,8 @@ reindexing_minimum_index_size: 1.gigabyte, reindexing_minimum_relative_bloat_size: 0.2, top_level_group_creation_enabled: true, - ci_partitions_size_limit: 100.gigabytes + ci_partitions_size_limit: 100.gigabytes, + ci_delete_pipelines_in_seconds_limit: ChronicDuration.parse('1 year') ) end end @@ -452,6 +453,22 @@ def many_usernames(num = 100) it { is_expected.not_to allow_value(nil).for(:math_rendering_limits_enabled) } + context 'with pipeline retention limits' do + it 'allows only integers' do + is_expected.to validate_numericality_of(:ci_delete_pipelines_in_seconds_limit) + .only_integer.is_greater_than_or_equal_to(1.day) + end + + it { is_expected.not_to allow_value(nil).for(:ci_delete_pipelines_in_seconds_limit) } + + describe '#ci_delete_pipelines_in_seconds_limit_human_readable=' do + it 'propagates values' do + expect { setting.ci_delete_pipelines_in_seconds_limit_human_readable = '1 month' } + .to change { setting.ci_delete_pipelines_in_seconds_limit }.to eq(ChronicDuration.parse('1 month')) + end + end + end + context 'when deactivate_dormant_users is enabled' do before do stub_application_setting(deactivate_dormant_users: true) @@ -2150,4 +2167,8 @@ def expect_invalid it { is_expected.not_to allow_value({ reindexing_minimum_index_size: "3" }).for(:database_reindexing) } it { is_expected.not_to allow_value({ reindexing_minimum_relative_bloat_size: true }).for(:database_reindexing) } end + + describe '#ci_delete_pipelines_in_seconds_limit_human_readable_long' do + it { expect(setting.ci_delete_pipelines_in_seconds_limit_human_readable_long).to eq('1 year') } + end end diff --git a/spec/models/project_ci_cd_setting_spec.rb b/spec/models/project_ci_cd_setting_spec.rb index df72110505260b..16641fa31bc1b2 100644 --- a/spec/models/project_ci_cd_setting_spec.rb +++ b/spec/models/project_ci_cd_setting_spec.rb @@ -45,6 +45,22 @@ .is_less_than_or_equal_to(ChronicDuration.parse('1 year')) .with_message('must be between 1 day and 1 year') end + + context 'with custom delete_pipelines_in_seconds limits' do + let(:limit) { ChronicDuration.parse('3 years, 2 months, 1 day') } + + before do + stub_application_setting(ci_delete_pipelines_in_seconds_limit: limit) + end + + it 'validates delete_pipelines_in_seconds' do + is_expected.to validate_numericality_of(:delete_pipelines_in_seconds) + .only_integer + .is_greater_than_or_equal_to(ChronicDuration.parse('1 day')) + .is_less_than_or_equal_to(limit) + .with_message('must be between 1 day and 38 months 16 days 18 hours') + end + end end describe '#pipeline_variables_minimum_override_role' do -- GitLab From 9443480f08ad4b5cdba281be71fce1617b3d1020 Mon Sep 17 00:00:00 2001 From: Marius Bobin Date: Mon, 28 Apr 2025 12:11:54 +0300 Subject: [PATCH 2/2] Apply 1 suggestion(s) to 1 file(s) --- doc/administration/instance_limits.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index cf2af50757ab7e..0399aebf4b45a2 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -991,7 +991,7 @@ ApplicationSetting.update(ci_partitions_size_limit: 20.gigabytes) {{< history >}} -- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/189131) in GitLab 18.0. +- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/189191) in GitLab 18.0. {{< /history >}} -- GitLab