From 597db2be246130e7bd2d8625e8471ed2ba0a79cc Mon Sep 17 00:00:00 2001 From: Marius Bobin Date: Thu, 24 Apr 2025 11:08:48 +0300 Subject: [PATCH 1/4] Add application settings to manage CI/CD partitions Changelog: added --- app/helpers/application_settings_helper.rb | 1 + app/models/application_setting.rb | 4 +++- .../application_setting_implementation.rb | 1 + .../application_setting_ci_cd_settings.json | 16 ++++++++++++++++ doc/administration/instance_limits.md | 18 ++++++++++++++++++ doc/api/settings.md | 3 ++- spec/models/application_setting_spec.rb | 4 +++- 7 files changed, 44 insertions(+), 3 deletions(-) create mode 100644 app/validators/json_schemas/application_setting_ci_cd_settings.json diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 73b0a8e1b035fa..d9e748d7b27cc4 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -268,6 +268,7 @@ def visible_attributes :autocomplete_users_limit, :autocomplete_users_unauthenticated_limit, :ci_job_live_trace_enabled, + :ci_partitions_size_limit, :concurrent_github_import_jobs_limit, :concurrent_bitbucket_import_jobs_limit, :concurrent_bitbucket_server_import_jobs_limit, diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 98d49ef05cb6cf..92e0f5e652278d 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -521,9 +521,11 @@ def self.kroki_formats_attributes if: ->(setting) { setting.external_auth_client_cert.present? } jsonb_accessor :ci_cd_settings, - ci_job_live_trace_enabled: [:boolean, { default: false }] + ci_job_live_trace_enabled: [:boolean, { default: false }], + ci_partitions_size_limit: [::Gitlab::Database::Type::JsonbInteger.new, { default: 100.gigabytes }] validate :validate_object_storage_for_live_trace_configuration, if: -> { ci_job_live_trace_enabled? } + validates :ci_partitions_size_limit, presence: true, numericality: { only_integer: true, greater_than: 0 } validates :default_ci_config_path, format: { without: %r{(\.{2}|\A/)}, message: N_('cannot include leading slash or directory traversal.') }, diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index adb29cffa91975..cd71001f8ea1e4 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -48,6 +48,7 @@ def defaults # rubocop:disable Metrics/AbcSize autocomplete_users_unauthenticated_limit: 100, ci_job_live_trace_enabled: false, ci_max_total_yaml_size_bytes: 314572800, # max_yaml_size_bytes * ci_max_includes = 2.megabyte * 150 + ci_partitions_size_limit: 100.gigabytes, commit_email_hostname: default_commit_email_hostname, container_expiration_policies_enable_historic_entries: false, container_registry_features: [], diff --git a/app/validators/json_schemas/application_setting_ci_cd_settings.json b/app/validators/json_schemas/application_setting_ci_cd_settings.json new file mode 100644 index 00000000000000..2eaf126ad9bf1c --- /dev/null +++ b/app/validators/json_schemas/application_setting_ci_cd_settings.json @@ -0,0 +1,16 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "Application settings for the CI/CD domain", + "type": "object", + "additionalProperties": false, + "properties": { + "ci_job_live_trace_enabled": { + "type": "boolean", + "description": "Enable cloud native architecture for storing job logs." + }, + "ci_partitions_size_limit": { + "type": "integer", + "description": "Maximum table size limit for a CI partition used for creating new partitions." + } + } +} diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index b0d2675bf197da..5b5740839749b0 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -969,6 +969,24 @@ To set this limit to 100 KB on a GitLab Self-Managed instance, run the following Plan.default.actual_limits.update!(ci_job_annotations_size: 100.kilobytes) ``` +### Maximum database partition size for CI/CD tables + +{{< history >}} + +- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38337) in GitLab 18.0. + +{{< /history >}} + +The maximum amount of disk space, in bytes, that can be used by a partition of a partitioned table, +before new partitions are automatically created. Defaults to 100 GB. + +You can change this limit by using the [GitLab Rails console](operations/rails_console.md#starting-a-rails-console-session). +To change the limit, update `ci_partitions_size_limit` with the new value. For example, to set it to 20 GB: + +```ruby +ApplicationSetting.update(ci_partitions_size_limit: 20.gigabytes) +``` + ## Instance monitoring and metrics ### Limit inbound incident management alerts diff --git a/doc/api/settings.md b/doc/api/settings.md index 857e19c5b1793a..2e20ba6409e7b6 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -465,9 +465,10 @@ to configure other related settings. These requirements are | `bulk_import_max_download_file_size` | integer | no | Maximum download file size when importing from source GitLab instances by direct transfer. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/384976) in GitLab 16.3. | | `can_create_group` | boolean | no | Indicates whether users can create top-level groups. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/367754) in GitLab 15.5. Defaults to `true`. | | `check_namespace_plan` | boolean | no | Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. Premium and Ultimate only. | -| `ci_job_live_trace_enabled` | boolean | no | Turns on incremental logging for job logs. When turned on, archived job logs are incrementally uploaded to object storage. Object storage must be configured. You can also configure this setting in the [**Admin** area](../administration/settings/continuous_integration.md#incremental-logging). | +| `ci_job_live_trace_enabled` | boolean | no | Turns on incremental logging for job logs. When turned on, archived job logs are incrementally uploaded to object storage. Object storage must be configured. You can also configure this setting in the [**Admin** area](../administration/settings/continuous_integration.md#incremental-logging). | | `ci_max_total_yaml_size_bytes` | integer | no | The maximum amount of memory, in bytes, that can be allocated for the pipeline configuration, with all included YAML configuration files. | | `ci_max_includes` | integer | no | The [maximum number of includes](../administration/settings/continuous_integration.md#maximum-includes) per pipeline. Default is `150`. | +| `ci_partitions_size_limit` | integer | no | The maximum amount of disk space, in bytes, that can be used by a database partition for the CI tables before creating new partitions. Default is `100 GB`. | | `concurrent_github_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the GitHub importer. Default is 1000. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.11. | | `concurrent_bitbucket_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the Bitbucket Cloud importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.11. | | `concurrent_bitbucket_server_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the Bitbucket Server importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.11. | diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 17a76357580255..799ff70640c99c 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -95,7 +95,8 @@ }, reindexing_minimum_index_size: 1.gigabyte, reindexing_minimum_relative_bloat_size: 0.2, - top_level_group_creation_enabled: true + top_level_group_creation_enabled: true, + ci_partitions_size_limit: 100.gigabytes ) end end @@ -384,6 +385,7 @@ def many_usernames(num = 100) autocomplete_users_limit autocomplete_users_unauthenticated_limit bulk_import_concurrent_pipeline_batch_limit + ci_partitions_size_limit code_suggestions_api_rate_limit concurrent_bitbucket_import_jobs_limit concurrent_bitbucket_server_import_jobs_limit -- GitLab From 8a0be3ab061d36a231ca6cc0a971cd3257b79359 Mon Sep 17 00:00:00 2001 From: Marius Bobin Date: Thu, 24 Apr 2025 11:23:35 +0300 Subject: [PATCH 2/4] Use application settings partitions limit --- app/models/ci/partition.rb | 5 +---- app/services/ci/partitions/create_service.rb | 5 +---- app/services/ci/partitions/sync_service.rb | 5 +---- spec/models/ci/partition_spec.rb | 6 ++++++ .../ci/partitions/create_service_spec.rb | 16 +--------------- .../ci/partitions/sync_service_spec.rb | 19 +------------------ 6 files changed, 11 insertions(+), 45 deletions(-) diff --git a/app/models/ci/partition.rb b/app/models/ci/partition.rb index 72a3a02a8a7f14..4d9640aafed0cd 100644 --- a/app/models/ci/partition.rb +++ b/app/models/ci/partition.rb @@ -2,9 +2,6 @@ module Ci class Partition < Ci::ApplicationRecord - MAX_PARTITION_SIZE = 100.gigabytes - GSTG_PARTITION_SIZE = 4.gigabytes - INITIAL_PARTITION_VALUE = 100 LATEST_PARTITION_VALUE = 102 DEFAULT_PARTITION_VALUES = (INITIAL_PARTITION_VALUE..LATEST_PARTITION_VALUE).to_a.freeze @@ -63,7 +60,7 @@ def provisioning(partition_id) end end - def above_threshold?(threshold) + def above_threshold?(threshold = ::Gitlab::CurrentSettings.ci_partitions_size_limit) with_ci_connection do Gitlab::Database::PostgresPartition .with_parent_tables(parent_table_names) diff --git a/app/services/ci/partitions/create_service.rb b/app/services/ci/partitions/create_service.rb index 52bfb08c9fe665..44796a62250a7d 100644 --- a/app/services/ci/partitions/create_service.rb +++ b/app/services/ci/partitions/create_service.rb @@ -25,10 +25,7 @@ def should_create_next? end def above_threshold? - threshold = Ci::Partition::MAX_PARTITION_SIZE - threshold = Ci::Partition::GSTG_PARTITION_SIZE if Gitlab.staging? - - partition.above_threshold?(threshold) + partition.above_threshold? end def headroom_available? diff --git a/app/services/ci/partitions/sync_service.rb b/app/services/ci/partitions/sync_service.rb index c70f9c5d294804..9756ecc6fc873d 100644 --- a/app/services/ci/partitions/sync_service.rb +++ b/app/services/ci/partitions/sync_service.rb @@ -24,10 +24,7 @@ def execute attr_reader :partition def above_threshold? - threshold = Ci::Partition::MAX_PARTITION_SIZE - threshold = Ci::Partition::GSTG_PARTITION_SIZE if Gitlab.staging? - - partition.above_threshold?(threshold) + partition.above_threshold? end def sync_available_partitions_statuses! diff --git a/spec/models/ci/partition_spec.rb b/spec/models/ci/partition_spec.rb index 079ba7fa2808de..6ad3dd5ec5bf1d 100644 --- a/spec/models/ci/partition_spec.rb +++ b/spec/models/ci/partition_spec.rb @@ -181,6 +181,12 @@ it { is_expected.to eq(false) } end + + context 'with default value' do + subject(:above_threshold) { ci_partition.above_threshold? } + + it { is_expected.to eq(false) } + end end describe '#all_partitions_exist?' do diff --git a/spec/services/ci/partitions/create_service_spec.rb b/spec/services/ci/partitions/create_service_spec.rb index 7618488dd572a6..410f87b234bc1c 100644 --- a/spec/services/ci/partitions/create_service_spec.rb +++ b/spec/services/ci/partitions/create_service_spec.rb @@ -47,7 +47,7 @@ context 'when database_partition sizes are above the threshold' do before do - stub_const("Ci::Partition::MAX_PARTITION_SIZE", 1.byte) + stub_application_setting(ci_partitions_size_limit: 1.byte) end context 'when no more headroom available' do @@ -64,20 +64,6 @@ it 'uses partition threshold' do expect(ci_partition) .to receive(:above_threshold?) - .with(Ci::Partition::MAX_PARTITION_SIZE) - .and_call_original - - expect { execute_service }.not_to change { Ci::Partition.count } - end - end - - context 'when executed on staging' do - it 'uses smaller threshold' do - expect(Gitlab).to receive(:staging?).and_return(true) - - expect(ci_partition) - .to receive(:above_threshold?) - .with(Ci::Partition::GSTG_PARTITION_SIZE) .and_call_original expect { execute_service }.not_to change { Ci::Partition.count } diff --git a/spec/services/ci/partitions/sync_service_spec.rb b/spec/services/ci/partitions/sync_service_spec.rb index cc7c2e74dc33ac..31b04cc6ad5d65 100644 --- a/spec/services/ci/partitions/sync_service_spec.rb +++ b/spec/services/ci/partitions/sync_service_spec.rb @@ -47,28 +47,11 @@ allow_next_found_instance_of(Ci::Partition) do |partition| allow(partition).to receive(:all_partitions_exist?).and_return(true) end - end - - it 'updates ci_partitions statuses', :aggregate_failures do - expect(ci_partition).to receive(:above_threshold?).with(Ci::Partition::MAX_PARTITION_SIZE).and_return(true) - - expect { execute_service } - .to change { ci_partition.reload.status }.from(current_status).to(active_status) - .and change { next_ci_partition.reload.status }.from(preparing_status).to(current_status) - end - end - context 'when executed on staging' do - before do - allow_next_found_instance_of(Ci::Partition) do |partition| - allow(partition).to receive(:all_partitions_exist?).and_return(true) - end + stub_application_setting(ci_partitions_size_limit: 1.byte) end it 'updates ci_partitions statuses', :aggregate_failures do - expect(Gitlab).to receive(:staging?).and_return(true) - expect(ci_partition).to receive(:above_threshold?).with(Ci::Partition::GSTG_PARTITION_SIZE).and_return(true) - expect { execute_service } .to change { ci_partition.reload.status }.from(current_status).to(active_status) .and change { next_ci_partition.reload.status }.from(preparing_status).to(current_status) -- GitLab From 5e50cadf30e76f67ffde54fe3ac096dc301fed38 Mon Sep 17 00:00:00 2001 From: Marius Bobin Date: Thu, 24 Apr 2025 11:27:08 +0300 Subject: [PATCH 3/4] Apply 1 suggestion(s) to 1 file(s) --- doc/administration/instance_limits.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/administration/instance_limits.md b/doc/administration/instance_limits.md index 5b5740839749b0..a6dc0b97a8f506 100644 --- a/doc/administration/instance_limits.md +++ b/doc/administration/instance_limits.md @@ -973,7 +973,7 @@ Plan.default.actual_limits.update!(ci_job_annotations_size: 100.kilobytes) {{< history >}} -- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/38337) in GitLab 18.0. +- [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/189131) in GitLab 18.0. {{< /history >}} -- GitLab From a599e60da89bb580fce3074cdae509a7b486b492 Mon Sep 17 00:00:00 2001 From: Marius Bobin Date: Thu, 24 Apr 2025 14:17:44 +0300 Subject: [PATCH 4/4] Fix threshold tests --- spec/services/ci/partitions/sync_service_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/services/ci/partitions/sync_service_spec.rb b/spec/services/ci/partitions/sync_service_spec.rb index 31b04cc6ad5d65..737287008e794f 100644 --- a/spec/services/ci/partitions/sync_service_spec.rb +++ b/spec/services/ci/partitions/sync_service_spec.rb @@ -47,11 +47,11 @@ allow_next_found_instance_of(Ci::Partition) do |partition| allow(partition).to receive(:all_partitions_exist?).and_return(true) end - - stub_application_setting(ci_partitions_size_limit: 1.byte) end it 'updates ci_partitions statuses', :aggregate_failures do + expect(ci_partition).to receive(:above_threshold?).and_return(true) + expect { execute_service } .to change { ci_partition.reload.status }.from(current_status).to(active_status) .and change { next_ci_partition.reload.status }.from(preparing_status).to(current_status) -- GitLab