From 25e8cc274b17cf5d8b5bc57bf3fa3ff332360f74 Mon Sep 17 00:00:00 2001 From: Rodrigo Tomonari Date: Mon, 5 Feb 2024 20:05:10 -0300 Subject: [PATCH 01/10] Update importers to use application settings to limit number of jobs Move hardcode limit of import jobs to an application settings on GitHub Import, BitBucketCloud Import and BitBucketServer Import Changelog: changed --- app/helpers/application_settings_helper.rb | 3 +++ app/models/application_setting.rb | 3 +++ .../_import_and_export.html.haml | 9 +++++++++ ...ent_import_jobs_to_application_settings.rb | 13 +++++++++++++ db/schema_migrations/20240228213914 | 1 + db/structure.sql | 3 +++ .../settings/import_and_export_settings.md | 19 +++++++++++++++++++ doc/api/settings.md | 13 +++++++++++-- lib/api/settings.rb | 3 +++ .../importers/pull_requests_importer.rb | 6 ++++++ .../bitbucket_import/parallel_scheduling.rb | 8 +++++--- .../importers/pull_requests_importer.rb | 14 +++++++++----- .../parallel_scheduling.rb | 8 +++++--- .../importer/pull_requests_importer.rb | 2 ++ .../github_import/job_delay_calculator.rb | 4 +--- locale/gitlab.pot | 9 +++++++++ .../parallel_scheduling_spec.rb | 2 +- .../parallel_scheduling_spec.rb | 2 +- .../job_delay_calculator_spec.rb | 12 ++++-------- spec/models/application_setting_spec.rb | 3 +++ spec/requests/api/settings_spec.rb | 11 ++++++++++- .../rescheduling_methods_spec.rb | 2 +- 22 files changed, 122 insertions(+), 28 deletions(-) create mode 100644 db/migrate/20240228213914_add_concurrent_import_jobs_to_application_settings.rb create mode 100644 db/schema_migrations/20240228213914 diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 540f93611c77d9..242bde1fb69747 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -232,6 +232,9 @@ def visible_attributes :authorized_keys_enabled, :auto_devops_enabled, :auto_devops_domain, + :concurrent_github_import_jobs_limit, + :concurrent_bitbucket_import_jobs_limit, + :concurrent_bitbucket_server_import_jobs_limit, :container_expiration_policies_enable_historic_entries, :container_registry_expiration_policies_caching, :container_registry_token_expire_delay, diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index 702d081fda93d5..a406bbfa96210b 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -522,6 +522,9 @@ def self.kroki_formats_attributes with_options(numericality: { only_integer: true, greater_than: 0 }) do validates :bulk_import_concurrent_pipeline_batch_limit, + :concurrent_github_import_jobs_limit, + :concurrent_bitbucket_import_jobs_limit, + :concurrent_bitbucket_server_import_jobs_limit, :container_registry_token_expire_delay, :housekeeping_optimize_repository_period, :inactive_projects_delete_after_months, diff --git a/app/views/admin/application_settings/_import_and_export.html.haml b/app/views/admin/application_settings/_import_and_export.html.haml index 8e321406bf8e1d..615cb8bdf900d4 100644 --- a/app/views/admin/application_settings/_import_and_export.html.haml +++ b/app/views/admin/application_settings/_import_and_export.html.haml @@ -40,4 +40,13 @@ = f.label :decompress_archive_file_timeout, s_('Import|Timeout for decompressing archived files (seconds)'), class: 'label-light' = f.number_field :decompress_archive_file_timeout, class: 'form-control gl-form-input', title: s_('Import|Timeout for decompressing archived files.'), data: { toggle: 'tooltip', container: 'body' } %span.form-text.text-muted= _('Set to 0 to disable timeout.') + .form-group + = f.label :concurrent_github_import_jobs_limit, s_('Import|GitHub Importer maximum number of simultaneous import jobs'), class: 'label-light' + = f.number_field :concurrent_github_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|GitHub Importer maximum number of simultaneous import jobs'), data: { toggle: 'tooltip', container: 'body' } + .form-group + = f.label :concurrent_bitbucket_import_jobs_limit, s_('Import|Bitbucket Cloud Importer maximum number of simultaneous import jobs'), class: 'label-light' + = f.number_field :concurrent_bitbucket_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|Bitbucket Cloud Importer maximum number of simultaneous import jobs'), data: { toggle: 'tooltip', container: 'body' } + .form-group + = f.label :concurrent_bitbucket_server_import_jobs_limit, s_('Import|Bitbucket Server Importer maximum number of simultaneous import jobs'), class: 'label-light' + = f.number_field :concurrent_bitbucket_server_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|Bitbucket Server Importer maximum number of simultaneous import jobs'), data: { toggle: 'tooltip', container: 'body' } = f.submit _('Save changes'), pajamas_button: true diff --git a/db/migrate/20240228213914_add_concurrent_import_jobs_to_application_settings.rb b/db/migrate/20240228213914_add_concurrent_import_jobs_to_application_settings.rb new file mode 100644 index 00000000000000..4491bafecdf5c0 --- /dev/null +++ b/db/migrate/20240228213914_add_concurrent_import_jobs_to_application_settings.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class AddConcurrentImportJobsToApplicationSettings < Gitlab::Database::Migration[2.2] + milestone '16.10' + enable_lock_retries! + + def change + add_column :application_settings, :concurrent_github_import_jobs_limit, :integer, default: 1000, null: false + add_column :application_settings, :concurrent_bitbucket_import_jobs_limit, :integer, default: 100, null: false + add_column :application_settings, :concurrent_bitbucket_server_import_jobs_limit, :integer, default: 100, + null: false + end +end diff --git a/db/schema_migrations/20240228213914 b/db/schema_migrations/20240228213914 new file mode 100644 index 00000000000000..36923a2dbb8c96 --- /dev/null +++ b/db/schema_migrations/20240228213914 @@ -0,0 +1 @@ +ad95ee3e8837ea35e1e7ae39ab75fc609016d228a48b4abed669862b567d32c2 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 4ad48e8acdefb2..978cb95d0928f5 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -4120,6 +4120,9 @@ CREATE TABLE application_settings ( asciidoc_max_includes smallint DEFAULT 32 NOT NULL, clickhouse jsonb DEFAULT '{}'::jsonb NOT NULL, include_optional_metrics_in_service_ping boolean DEFAULT true NOT NULL, + concurrent_github_import_jobs_limit integer DEFAULT 1000 NOT NULL, + concurrent_bitbucket_import_jobs_limit integer DEFAULT 100 NOT NULL, + concurrent_bitbucket_server_import_jobs_limit integer DEFAULT 100 NOT NULL, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), CONSTRAINT app_settings_container_registry_pre_import_tags_rate_positive CHECK ((container_registry_pre_import_tags_rate >= (0)::numeric)), CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)), diff --git a/doc/administration/settings/import_and_export_settings.md b/doc/administration/settings/import_and_export_settings.md index a6030d35b572c9..68acd910edf628 100644 --- a/doc/administration/settings/import_and_export_settings.md +++ b/doc/administration/settings/import_and_export_settings.md @@ -157,3 +157,22 @@ To modify the maximum decompressed file size for imports in GitLab: 1. Select **Settings > General**. 1. Expand **Import and export settings**. 1. Set another value for **Timeout for decompressing archived files (seconds)**. + +## Maximum number of simultaneous import jobs + +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. + +When you import a project using [GitHub Importer](../../user/project/import/github.md), +[Bitbucket Cloud Importer](../../user/project/import/bitbucket.md) or [Bitbucket Server Importer](../../user/project/import/bitbucket_server.md), +you can specify the maximum number of import jobs that are executed simultaneously. + +The default limit for GitHub Importer is set to 1000 jobs, whereas for Bitbucket Cloud Importer and Bitbucket Server Importer, +it's set to 100. However, it's worth noting that the Bitbucket importers currently have a relatively low default limit. +This is because thorough testing is yet to be conducted to ascertain an appropriate and reasonable default limit. + +To modify this setting: + +1. On the left sidebar, at the bottom, select **Admin Area**. +1. Select **Settings > General**. +1. Expand **Import and export settings**. +1. Set another value for **maximum number of simultaneous import jobs** for the desired importer. diff --git a/doc/api/settings.md b/doc/api/settings.md index 7fb30deb79abee..c1ae1b1bf52e88 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -132,7 +132,10 @@ Example response: "bulk_import_max_download_file_size": 5120, "project_jobs_api_rate_limit": 600, "security_txt_content": null, - "bulk_import_concurrent_pipeline_batch_limit": 25 + "bulk_import_concurrent_pipeline_batch_limit": 25, + "concurrent_github_import_jobs_limit": 1000, + "concurrent_bitbucket_import_jobs_limit": 100, + "concurrent_bitbucket_server_import_jobs_limit": 100 } ``` @@ -283,7 +286,10 @@ Example response: "project_jobs_api_rate_limit": 600, "security_txt_content": null, "bulk_import_concurrent_pipeline_batch_limit": 25, - "downstream_pipeline_trigger_limit_per_project_user_sha": 0 + "downstream_pipeline_trigger_limit_per_project_user_sha": 0, + "concurrent_github_import_jobs_limit": 1000, + "concurrent_bitbucket_import_jobs_limit": 100, + "concurrent_bitbucket_server_import_jobs_limit": 100 } ``` @@ -353,6 +359,9 @@ listed in the descriptions of the relevant settings. | `check_namespace_plan` | boolean | no | Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. Premium and Ultimate only. | | `ci_max_total_yaml_size_bytes` | integer | no | The maximum amount of memory, in bytes, that can be allocated for the pipeline configuration, with all included YAML configuration files. | | `ci_max_includes` | integer | no | The [maximum number of includes](../administration/settings/continuous_integration.md#maximum-includes) per pipeline. Default is `150`. | +| `concurrent_github_import_jobs_limit` | integer | no | GitHub Importer maximum number of simultaneous import jobs. Default is 1000. | +| `concurrent_bitbucket_import_jobs_limit` | integer | no | Bitbucket Cloud Importer maximum number of simultaneous import jobs. Default is 100. | +| `concurrent_bitbucket_server_import_jobs_limit` | integer | no | Bitbucket Server Importer maximum number of simultaneous import jobs. Default is 100. | | `commit_email_hostname` | string | no | Custom hostname (for private commit emails). | | `container_expiration_policies_enable_historic_entries` | boolean | no | Enable [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#enable-the-cleanup-policy) for all projects. | | `container_registry_cleanup_tags_service_max_list_size` | integer | no | The maximum number of tags that can be deleted in a single execution of [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#set-cleanup-limits-to-conserve-resources). | diff --git a/lib/api/settings.rb b/lib/api/settings.rb index 765fa6df154ec0..4d298b60800b2b 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -216,6 +216,9 @@ def filter_attributes_using_license(attrs) optional :bulk_import_concurrent_pipeline_batch_limit, type: Integer, desc: 'Maximum simultaneous Direct Transfer pipeline batches to process' optional :bulk_import_enabled, type: Boolean, desc: 'Enable migrating GitLab groups and projects by direct transfer' optional :bulk_import_max_download_file, type: Integer, desc: 'Maximum download file size in MB when importing from source GitLab instances by direct transfer' + optional :concurrent_github_import_jobs_limit, type: Integer, desc: 'Github Importer maximum number of simultaneous import jobs' + optional :concurrent_bitbucket_import_jobs_limit, type: Integer, desc: 'Bitbucket Cloud Importer maximum number of simultaneous import jobs' + optional :concurrent_bitbucket_server_import_jobs_limit, type: Integer, desc: 'Bitbucket Server Importer maximum number of simultaneous import jobs' optional :allow_runner_registration_token, type: Boolean, desc: 'Allow registering runners using a registration token' optional :ci_max_includes, type: Integer, desc: 'Maximum number of includes per pipeline' optional :security_policy_global_group_approvers_enabled, type: Boolean, desc: 'Query scan result policy approval groups globally' diff --git a/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb index eedb89c2d49d0c..99bf434a396ac5 100644 --- a/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb +++ b/lib/gitlab/bitbucket_import/importers/pull_requests_importer.rb @@ -42,6 +42,12 @@ def collection_method def id_for_already_enqueued_cache(object) object.iid end + + # To avoid overloading Gitaly, we use a smaller limit for pull requests than the one defined in the + # application settings. + def concurrent_import_jobs_limit + 100 + end end end end diff --git a/lib/gitlab/bitbucket_import/parallel_scheduling.rb b/lib/gitlab/bitbucket_import/parallel_scheduling.rb index 09d1eb683195b9..1cc5855be76a3f 100644 --- a/lib/gitlab/bitbucket_import/parallel_scheduling.rb +++ b/lib/gitlab/bitbucket_import/parallel_scheduling.rb @@ -16,8 +16,6 @@ module ParallelScheduling JOB_WAITER_CACHE_KEY = 'bitbucket-importer/job-waiter/%{project}/%{collection}' - BATCH_SIZE = 100 - # project - An instance of `Project`. def initialize(project) @project = project @@ -77,7 +75,7 @@ def mark_as_enqueued(object) def calculate_job_delay(job_index) runtime = Time.current - job_started_at - multiplier = (job_index / BATCH_SIZE.to_f) + multiplier = (job_index / concurrent_import_jobs_limit.to_f) (multiplier * 1.minute) + 1.second - runtime end @@ -85,6 +83,10 @@ def calculate_job_delay(job_index) def job_started_at @job_started_at ||= Time.current end + + def concurrent_import_jobs_limit + Gitlab::CurrentSettings.concurrent_bitbucket_import_jobs_limit + end end end end diff --git a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb index 5d92a2565a70f2..32939aeb03ed77 100644 --- a/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/pull_requests_importer.rb @@ -6,20 +6,17 @@ module Importers class PullRequestsImporter include ParallelScheduling - # Reduce fetch limit (from 100) to avoid Gitlab::Git::ResourceExhaustedError - PULL_REQUESTS_BATCH_SIZE = 50 - def execute page = 1 loop do log_info( import_stage: 'import_pull_requests', - message: "importing page #{page} using batch-size #{PULL_REQUESTS_BATCH_SIZE}" + message: "importing page #{page} using batch-size #{concurrent_import_jobs_limit}" ) pull_requests = client.pull_requests( - project_key, repository_slug, page_offset: page, limit: PULL_REQUESTS_BATCH_SIZE + project_key, repository_slug, page_offset: page, limit: concurrent_import_jobs_limit ).to_a break if pull_requests.empty? @@ -110,6 +107,13 @@ def source_branch_commit(source_branch_sha, pull_request) def target_branch_commit(target_branch_sha) [target_branch_sha, ':refs/keep-around/', target_branch_sha].join end + + # To avoid overloading Gitaly, we use a smaller limit for pull requests than the one defined in the + # application settings. + def concurrent_import_jobs_limit + # Reduce fetch limit (from 100) to avoid Gitlab::Git::ResourceExhaustedError + 50 + end end end end diff --git a/lib/gitlab/bitbucket_server_import/parallel_scheduling.rb b/lib/gitlab/bitbucket_server_import/parallel_scheduling.rb index 503dfc37ac2219..fbaa0916dc6ab8 100644 --- a/lib/gitlab/bitbucket_server_import/parallel_scheduling.rb +++ b/lib/gitlab/bitbucket_server_import/parallel_scheduling.rb @@ -15,8 +15,6 @@ module ParallelScheduling JOB_WAITER_CACHE_KEY = 'bitbucket-server-importer/job-waiter/%{project}/%{collection}' - BATCH_SIZE = 100 - # project - An instance of `Project`. def initialize(project) @project = project @@ -84,7 +82,7 @@ def mark_as_processed(object) def calculate_job_delay(job_index) runtime = Time.current - job_started_at - multiplier = (job_index / BATCH_SIZE.to_f) + multiplier = (job_index / concurrent_import_jobs_limit.to_f) (multiplier * 1.minute) + 1.second - runtime end @@ -93,6 +91,10 @@ def job_started_at @job_started_at ||= Time.current end + def concurrent_import_jobs_limit + Gitlab::CurrentSettings.concurrent_bitbucket_server_import_jobs_limit + end + def track_import_failure!(project, exception:, **args) Gitlab::Import::ImportFailureService.track( project_id: project.id, diff --git a/lib/gitlab/github_import/importer/pull_requests_importer.rb b/lib/gitlab/github_import/importer/pull_requests_importer.rb index 671e023e90ba01..cffe984612cd07 100644 --- a/lib/gitlab/github_import/importer/pull_requests_importer.rb +++ b/lib/gitlab/github_import/importer/pull_requests_importer.rb @@ -74,6 +74,8 @@ def collection_options { state: 'all', sort: 'created', direction: 'asc' } end + # To avoid overloading Gitaly, we use a smaller limit for pull requests than the one defined in the + # application settings. def parallel_import_batch { size: 200, delay: 1.minute } end diff --git a/lib/gitlab/github_import/job_delay_calculator.rb b/lib/gitlab/github_import/job_delay_calculator.rb index a456e198afdee1..a9aa3887f58882 100644 --- a/lib/gitlab/github_import/job_delay_calculator.rb +++ b/lib/gitlab/github_import/job_delay_calculator.rb @@ -7,9 +7,7 @@ module GithubImport module JobDelayCalculator # Default batch settings for parallel import (can be redefined in Importer/Worker classes) def parallel_import_batch - batch_size = Feature.enabled?(:github_import_increased_concurrent_workers, project.creator) ? 5000 : 1000 - - { size: batch_size, delay: 1.minute } + { size: Gitlab::CurrentSettings.concurrent_github_import_jobs_limit, delay: 1.minute } end private diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 0ec9466fe93078..3b20b0de9ad0f3 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -25795,9 +25795,18 @@ msgstr "" msgid "Import|An error occurred while fetching import details." msgstr "" +msgid "Import|Bitbucket Cloud Importer maximum number of simultaneous import jobs" +msgstr "" + +msgid "Import|Bitbucket Server Importer maximum number of simultaneous import jobs" +msgstr "" + msgid "Import|Failures for %{id}" msgstr "" +msgid "Import|GitHub Importer maximum number of simultaneous import jobs" +msgstr "" + msgid "Import|GitHub import details" msgstr "" diff --git a/spec/lib/gitlab/bitbucket_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/bitbucket_import/parallel_scheduling_spec.rb index 7aedc8e1466c5b..269ff6631e636b 100644 --- a/spec/lib/gitlab/bitbucket_import/parallel_scheduling_spec.rb +++ b/spec/lib/gitlab/bitbucket_import/parallel_scheduling_spec.rb @@ -19,7 +19,7 @@ def collection_method let(:importer) { importer_class.new(project) } before do - stub_const("#{described_class}::BATCH_SIZE", 2) + stub_application_setting(concurrent_bitbucket_import_jobs_limit: 2) end it 'returns an incremental delay', :freeze_time do diff --git a/spec/lib/gitlab/bitbucket_server_import/parallel_scheduling_spec.rb b/spec/lib/gitlab/bitbucket_server_import/parallel_scheduling_spec.rb index bfd701a6ea46ee..86ea83e830c3e3 100644 --- a/spec/lib/gitlab/bitbucket_server_import/parallel_scheduling_spec.rb +++ b/spec/lib/gitlab/bitbucket_server_import/parallel_scheduling_spec.rb @@ -19,7 +19,7 @@ def collection_method let(:importer) { importer_class.new(project) } before do - stub_const("#{described_class}::BATCH_SIZE", 2) + stub_application_setting(concurrent_bitbucket_server_import_jobs_limit: 2) end it 'returns an incremental delay', :freeze_time do diff --git a/spec/lib/gitlab/github_import/job_delay_calculator_spec.rb b/spec/lib/gitlab/github_import/job_delay_calculator_spec.rb index 3ddf8136dcf888..55e5965fd5d7d5 100644 --- a/spec/lib/gitlab/github_import/job_delay_calculator_spec.rb +++ b/spec/lib/gitlab/github_import/job_delay_calculator_spec.rb @@ -20,14 +20,10 @@ def initialize(project) describe "#parallel_import_batch" do subject { importer_class.new(project).parallel_import_batch } - it { is_expected.to eq({ size: 5000, delay: 1.minute }) } - - context 'when `github_import_increased_concurrent_workers` feature flag is disabled' do - before do - stub_feature_flags(github_import_increased_concurrent_workers: false) - end - - it { is_expected.to eq({ size: 1000, delay: 1.minute }) } + before do + stub_application_setting(concurrent_github_import_jobs_limit: 10) end + + it { is_expected.to eq({ size: 10, delay: 1.minute }) } end end diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index db6b987ca267c1..a5a2592a5101dc 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -31,6 +31,9 @@ it { expect(setting.members_delete_limit).to eq(60) } it { expect(setting.downstream_pipeline_trigger_limit_per_project_user_sha).to eq(0) } it { expect(setting.asciidoc_max_includes).to eq(32) } + it { expect(setting.concurrent_github_import_jobs_limit).to eq(1000) } + it { expect(setting.concurrent_bitbucket_import_jobs_limit).to eq(1000) } + it { expect(setting.concurrent_bitbucket_server_import_jobs_limit).to eq(1000) } end describe 'validations' do diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb index 018e6b9786418b..a146257159f073 100644 --- a/spec/requests/api/settings_spec.rb +++ b/spec/requests/api/settings_spec.rb @@ -96,6 +96,9 @@ expect(json_response['failed_login_attempts_unlock_period_in_minutes']).to be_nil expect(json_response['bulk_import_concurrent_pipeline_batch_limit']).to eq(25) expect(json_response['downstream_pipeline_trigger_limit_per_project_user_sha']).to eq(0) + expect(json_response['concurrent_github_import_jobs_limit']).to eq(1000) + expect(json_response['concurrent_bitbucket_import_jobs_limit']).to eq(1000) + expect(json_response['concurrent_bitbucket_server_import_jobs_limit']).to eq(1000) end end @@ -218,7 +221,10 @@ namespace_aggregation_schedule_lease_duration_in_seconds: 400, max_import_remote_file_size: 2, security_txt_content: nil, - downstream_pipeline_trigger_limit_per_project_user_sha: 300 + downstream_pipeline_trigger_limit_per_project_user_sha: 300, + concurrent_github_import_jobs_limit: 2, + concurrent_bitbucket_import_jobs_limit: 2, + concurrent_bitbucket_server_import_jobs_limit: 2 } expect(response).to have_gitlab_http_status(:ok) @@ -305,6 +311,9 @@ expect(json_response['security_txt_content']).to be(nil) expect(json_response['bulk_import_concurrent_pipeline_batch_limit']).to be(2) expect(json_response['downstream_pipeline_trigger_limit_per_project_user_sha']).to be(300) + expect(json_response['concurrent_github_import_jobs_limit']).to be(2) + expect(json_response['concurrent_bitbucket_import_jobs_limit']).to be(2) + expect(json_response['concurrent_bitbucket_server_import_jobs_limit']).to be(2) end end diff --git a/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb b/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb index 7b8c4fab0c67a1..9c641c7254f8fa 100644 --- a/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb +++ b/spec/workers/concerns/gitlab/github_import/rescheduling_methods_spec.rb @@ -76,7 +76,7 @@ def self.name expect(worker.class) .to receive(:perform_in) - .with(15.012, project.id, { 'number' => 2 }, '123') + .with(15.06, project.id, { 'number' => 2 }, '123') worker.perform(project.id, { 'number' => 2 }, '123') end -- GitLab From 38bab757b84cdaab2c8b43c1fd9392156ebcdfbe Mon Sep 17 00:00:00 2001 From: Rodrigo Tomonari Date: Mon, 4 Mar 2024 15:41:53 -0300 Subject: [PATCH 02/10] Delete feature flag definition --- .../github_import_increased_concurrent_workers.yml | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 config/feature_flags/development/github_import_increased_concurrent_workers.yml diff --git a/config/feature_flags/development/github_import_increased_concurrent_workers.yml b/config/feature_flags/development/github_import_increased_concurrent_workers.yml deleted file mode 100644 index 7e5adaadf62fed..00000000000000 --- a/config/feature_flags/development/github_import_increased_concurrent_workers.yml +++ /dev/null @@ -1,8 +0,0 @@ ---- -name: github_import_increased_concurrent_workers -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/137832 -rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/432891 -milestone: '16.7' -type: development -group: group::import and integrate -default_enabled: false -- GitLab From 8366749cfc1a1a3f4ac63545191ed9ce8af46f63 Mon Sep 17 00:00:00 2001 From: Rodrigo Tomonari Date: Mon, 4 Mar 2024 16:55:48 -0300 Subject: [PATCH 03/10] Fix settings spec --- spec/models/application_setting_spec.rb | 4 ++-- spec/requests/api/settings_spec.rb | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index a5a2592a5101dc..09d8743304b676 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -32,8 +32,8 @@ it { expect(setting.downstream_pipeline_trigger_limit_per_project_user_sha).to eq(0) } it { expect(setting.asciidoc_max_includes).to eq(32) } it { expect(setting.concurrent_github_import_jobs_limit).to eq(1000) } - it { expect(setting.concurrent_bitbucket_import_jobs_limit).to eq(1000) } - it { expect(setting.concurrent_bitbucket_server_import_jobs_limit).to eq(1000) } + it { expect(setting.concurrent_bitbucket_import_jobs_limit).to eq(100) } + it { expect(setting.concurrent_bitbucket_server_import_jobs_limit).to eq(100) } end describe 'validations' do diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb index a146257159f073..ea8907766e06d0 100644 --- a/spec/requests/api/settings_spec.rb +++ b/spec/requests/api/settings_spec.rb @@ -97,8 +97,8 @@ expect(json_response['bulk_import_concurrent_pipeline_batch_limit']).to eq(25) expect(json_response['downstream_pipeline_trigger_limit_per_project_user_sha']).to eq(0) expect(json_response['concurrent_github_import_jobs_limit']).to eq(1000) - expect(json_response['concurrent_bitbucket_import_jobs_limit']).to eq(1000) - expect(json_response['concurrent_bitbucket_server_import_jobs_limit']).to eq(1000) + expect(json_response['concurrent_bitbucket_import_jobs_limit']).to eq(100) + expect(json_response['concurrent_bitbucket_server_import_jobs_limit']).to eq(100) end end -- GitLab From c32b2e8d3f23cf537d6540d05d30e7f0727ba418 Mon Sep 17 00:00:00 2001 From: Rodrigo Tomonari Date: Tue, 5 Mar 2024 10:53:11 -0300 Subject: [PATCH 04/10] Update user docs --- .../settings/import_and_export_settings.md | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/doc/administration/settings/import_and_export_settings.md b/doc/administration/settings/import_and_export_settings.md index 68acd910edf628..35ec5699d50571 100644 --- a/doc/administration/settings/import_and_export_settings.md +++ b/doc/administration/settings/import_and_export_settings.md @@ -163,12 +163,16 @@ To modify the maximum decompressed file size for imports in GitLab: > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. When you import a project using [GitHub Importer](../../user/project/import/github.md), -[Bitbucket Cloud Importer](../../user/project/import/bitbucket.md) or [Bitbucket Server Importer](../../user/project/import/bitbucket_server.md), -you can specify the maximum number of import jobs that are executed simultaneously. +[Bitbucket Cloud Importer](../../user/project/import/bitbucket.md) or +[Bitbucket Server Importer](../../user/project/import/bitbucket_server.md), you +can specify the maximum number of import jobs that are executed simultaneously. -The default limit for GitHub Importer is set to 1000 jobs, whereas for Bitbucket Cloud Importer and Bitbucket Server Importer, -it's set to 100. However, it's worth noting that the Bitbucket importers currently have a relatively low default limit. -This is because thorough testing is yet to be conducted to ascertain an appropriate and reasonable default limit. +The default job limit for GitHub Importer is 1000, whereas for Bitbucket Cloud +Importer and Bitbucket Server Importer, it's set to 100. However, it's worth +noting that Bitbucket importers currently have a relatively low default limit. +This is because thorough testing is yet to be conducted to determine an +appropriate and reasonable default limit. Therefore, Self-managed +administrators are encouraged to experiment with an increased limit. To modify this setting: @@ -176,3 +180,7 @@ To modify this setting: 1. Select **Settings > General**. 1. Expand **Import and export settings**. 1. Set another value for **maximum number of simultaneous import jobs** for the desired importer. + +NOTE: +This settings is not applied when importing merge requests, as +for this resource, the limit is hardcoded to a low number not to overload Gitaly. -- GitLab From 1c3d324de5088bef1ec90e22c9cac9c23451c054 Mon Sep 17 00:00:00 2001 From: Evan Read Date: Wed, 6 Mar 2024 03:11:32 +0000 Subject: [PATCH 05/10] Apply technical writing suggestions --- .../_import_and_export.html.haml | 6 +++--- .../settings/import_and_export_settings.md | 19 ++++++++----------- doc/api/settings.md | 6 +++--- 3 files changed, 14 insertions(+), 17 deletions(-) diff --git a/app/views/admin/application_settings/_import_and_export.html.haml b/app/views/admin/application_settings/_import_and_export.html.haml index 615cb8bdf900d4..a80502b253c4a9 100644 --- a/app/views/admin/application_settings/_import_and_export.html.haml +++ b/app/views/admin/application_settings/_import_and_export.html.haml @@ -41,12 +41,12 @@ = f.number_field :decompress_archive_file_timeout, class: 'form-control gl-form-input', title: s_('Import|Timeout for decompressing archived files.'), data: { toggle: 'tooltip', container: 'body' } %span.form-text.text-muted= _('Set to 0 to disable timeout.') .form-group - = f.label :concurrent_github_import_jobs_limit, s_('Import|GitHub Importer maximum number of simultaneous import jobs'), class: 'label-light' + = f.label :concurrent_github_import_jobs_limit, s_('Import|Maximum number of simultaneous import jobs for GitHub importer'), class: 'label-light' = f.number_field :concurrent_github_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|GitHub Importer maximum number of simultaneous import jobs'), data: { toggle: 'tooltip', container: 'body' } .form-group - = f.label :concurrent_bitbucket_import_jobs_limit, s_('Import|Bitbucket Cloud Importer maximum number of simultaneous import jobs'), class: 'label-light' + = f.label :concurrent_bitbucket_import_jobs_limit, s_('Import|Maximum number of simultaneous import jobs for Bitbucket Cloud importer'), class: 'label-light' = f.number_field :concurrent_bitbucket_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|Bitbucket Cloud Importer maximum number of simultaneous import jobs'), data: { toggle: 'tooltip', container: 'body' } .form-group - = f.label :concurrent_bitbucket_server_import_jobs_limit, s_('Import|Bitbucket Server Importer maximum number of simultaneous import jobs'), class: 'label-light' + = f.label :concurrent_bitbucket_server_import_jobs_limit, s_('Import|Maximum number of simultaneous import jobs for Bitbucket Server importer'), class: 'label-light' = f.number_field :concurrent_bitbucket_server_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|Bitbucket Server Importer maximum number of simultaneous import jobs'), data: { toggle: 'tooltip', container: 'body' } = f.submit _('Save changes'), pajamas_button: true diff --git a/doc/administration/settings/import_and_export_settings.md b/doc/administration/settings/import_and_export_settings.md index 35ec5699d50571..985e4ce8f8430d 100644 --- a/doc/administration/settings/import_and_export_settings.md +++ b/doc/administration/settings/import_and_export_settings.md @@ -167,20 +167,17 @@ When you import a project using [GitHub Importer](../../user/project/import/gith [Bitbucket Server Importer](../../user/project/import/bitbucket_server.md), you can specify the maximum number of import jobs that are executed simultaneously. -The default job limit for GitHub Importer is 1000, whereas for Bitbucket Cloud -Importer and Bitbucket Server Importer, it's set to 100. However, it's worth -noting that Bitbucket importers currently have a relatively low default limit. -This is because thorough testing is yet to be conducted to determine an -appropriate and reasonable default limit. Therefore, Self-managed -administrators are encouraged to experiment with an increased limit. +The job limit is not applied when importing merge requests because there is a hard-coded limit for merge requests to avoid overloading servers. + +The default job limit is: + +- For the GitHub importer, 1000. +- For the Bitbucket Cloud and Bitbucket Server importer, 100. The Bitbucket importers have a low default limit because we haven't yet determined + a good default limit. Administrators of self-managed GitLab instances should experiment with a higher limit. To modify this setting: 1. On the left sidebar, at the bottom, select **Admin Area**. 1. Select **Settings > General**. 1. Expand **Import and export settings**. -1. Set another value for **maximum number of simultaneous import jobs** for the desired importer. - -NOTE: -This settings is not applied when importing merge requests, as -for this resource, the limit is hardcoded to a low number not to overload Gitaly. +1. Set another value for **Maximum number of simultaneous import jobs** for the desired importer. diff --git a/doc/api/settings.md b/doc/api/settings.md index c1ae1b1bf52e88..6fe434931a769d 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -359,9 +359,9 @@ listed in the descriptions of the relevant settings. | `check_namespace_plan` | boolean | no | Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. Premium and Ultimate only. | | `ci_max_total_yaml_size_bytes` | integer | no | The maximum amount of memory, in bytes, that can be allocated for the pipeline configuration, with all included YAML configuration files. | | `ci_max_includes` | integer | no | The [maximum number of includes](../administration/settings/continuous_integration.md#maximum-includes) per pipeline. Default is `150`. | -| `concurrent_github_import_jobs_limit` | integer | no | GitHub Importer maximum number of simultaneous import jobs. Default is 1000. | -| `concurrent_bitbucket_import_jobs_limit` | integer | no | Bitbucket Cloud Importer maximum number of simultaneous import jobs. Default is 100. | -| `concurrent_bitbucket_server_import_jobs_limit` | integer | no | Bitbucket Server Importer maximum number of simultaneous import jobs. Default is 100. | +| `concurrent_github_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the GitHub importer. Default is 1000. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | +| `concurrent_bitbucket_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the BitBucket Cloud importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | +| `concurrent_bitbucket_server_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the Bitbucket Server importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | | `commit_email_hostname` | string | no | Custom hostname (for private commit emails). | | `container_expiration_policies_enable_historic_entries` | boolean | no | Enable [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#enable-the-cleanup-policy) for all projects. | | `container_registry_cleanup_tags_service_max_list_size` | integer | no | The maximum number of tags that can be deleted in a single execution of [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#set-cleanup-limits-to-conserve-resources). | -- GitLab From 7ff0fe7f219375f6beb04ea3d6b0535cc42173ed Mon Sep 17 00:00:00 2001 From: Rodrigo Tomonari Date: Wed, 6 Mar 2024 00:14:50 -0300 Subject: [PATCH 06/10] Change attribute title and regenerate locale --- .../application_settings/_import_and_export.html.haml | 6 +++--- doc/api/settings.md | 2 +- locale/gitlab.pot | 9 +++++++++ 3 files changed, 13 insertions(+), 4 deletions(-) diff --git a/app/views/admin/application_settings/_import_and_export.html.haml b/app/views/admin/application_settings/_import_and_export.html.haml index a80502b253c4a9..e1c2d727342151 100644 --- a/app/views/admin/application_settings/_import_and_export.html.haml +++ b/app/views/admin/application_settings/_import_and_export.html.haml @@ -42,11 +42,11 @@ %span.form-text.text-muted= _('Set to 0 to disable timeout.') .form-group = f.label :concurrent_github_import_jobs_limit, s_('Import|Maximum number of simultaneous import jobs for GitHub importer'), class: 'label-light' - = f.number_field :concurrent_github_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|GitHub Importer maximum number of simultaneous import jobs'), data: { toggle: 'tooltip', container: 'body' } + = f.number_field :concurrent_github_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|Maximum number of simultaneous import jobs for GitHub importer'), data: { toggle: 'tooltip', container: 'body' } .form-group = f.label :concurrent_bitbucket_import_jobs_limit, s_('Import|Maximum number of simultaneous import jobs for Bitbucket Cloud importer'), class: 'label-light' - = f.number_field :concurrent_bitbucket_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|Bitbucket Cloud Importer maximum number of simultaneous import jobs'), data: { toggle: 'tooltip', container: 'body' } + = f.number_field :concurrent_bitbucket_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|Maximum number of simultaneous import jobs for Bitbucket Cloud importer'), data: { toggle: 'tooltip', container: 'body' } .form-group = f.label :concurrent_bitbucket_server_import_jobs_limit, s_('Import|Maximum number of simultaneous import jobs for Bitbucket Server importer'), class: 'label-light' - = f.number_field :concurrent_bitbucket_server_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|Bitbucket Server Importer maximum number of simultaneous import jobs'), data: { toggle: 'tooltip', container: 'body' } + = f.number_field :concurrent_bitbucket_server_import_jobs_limit, class: 'form-control gl-form-input', title: s_('Import|Maximum number of simultaneous import jobs for Bitbucket Server importer'), data: { toggle: 'tooltip', container: 'body' } = f.submit _('Save changes'), pajamas_button: true diff --git a/doc/api/settings.md b/doc/api/settings.md index 6fe434931a769d..5217a60eabb791 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -360,7 +360,7 @@ listed in the descriptions of the relevant settings. | `ci_max_total_yaml_size_bytes` | integer | no | The maximum amount of memory, in bytes, that can be allocated for the pipeline configuration, with all included YAML configuration files. | | `ci_max_includes` | integer | no | The [maximum number of includes](../administration/settings/continuous_integration.md#maximum-includes) per pipeline. Default is `150`. | | `concurrent_github_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the GitHub importer. Default is 1000. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | -| `concurrent_bitbucket_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the BitBucket Cloud importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | +| `concurrent_bitbucket_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the Bitbucket Cloud importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | | `concurrent_bitbucket_server_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the Bitbucket Server importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | | `commit_email_hostname` | string | no | Custom hostname (for private commit emails). | | `container_expiration_policies_enable_historic_entries` | boolean | no | Enable [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#enable-the-cleanup-policy) for all projects. | diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 3b20b0de9ad0f3..300fef0e5c92ec 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -25816,6 +25816,15 @@ msgstr "" msgid "Import|Maximum import remote file size (MiB)" msgstr "" +msgid "Import|Maximum number of simultaneous import jobs for Bitbucket Cloud importer" +msgstr "" + +msgid "Import|Maximum number of simultaneous import jobs for Bitbucket Server importer" +msgstr "" + +msgid "Import|Maximum number of simultaneous import jobs for GitHub importer" +msgstr "" + msgid "Import|Maximum remote file size for imports from external object storages. For example, AWS S3." msgstr "" -- GitLab From 533f4bd164b6b6ea530dab69d669abd2a3b484bd Mon Sep 17 00:00:00 2001 From: Rodrigo Tomonari Date: Wed, 6 Mar 2024 11:23:12 -0300 Subject: [PATCH 07/10] Fix locale file --- locale/gitlab.pot | 9 --------- 1 file changed, 9 deletions(-) diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 300fef0e5c92ec..7fb53e4ada1dfe 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -25795,18 +25795,9 @@ msgstr "" msgid "Import|An error occurred while fetching import details." msgstr "" -msgid "Import|Bitbucket Cloud Importer maximum number of simultaneous import jobs" -msgstr "" - -msgid "Import|Bitbucket Server Importer maximum number of simultaneous import jobs" -msgstr "" - msgid "Import|Failures for %{id}" msgstr "" -msgid "Import|GitHub Importer maximum number of simultaneous import jobs" -msgstr "" - msgid "Import|GitHub import details" msgstr "" -- GitLab From 5eb6eabc388e0734676acbfcb358722a3cde87c7 Mon Sep 17 00:00:00 2001 From: Rodrigo Tomonari Date: Wed, 20 Mar 2024 17:49:19 -0300 Subject: [PATCH 08/10] Use rate_limit column --- app/models/application_setting.rb | 5 ++++- .../application_setting_rate_limits.json | 15 +++++++++++++++ ...current_import_jobs_to_application_settings.rb | 13 ------------- db/structure.sql | 3 --- 4 files changed, 19 insertions(+), 17 deletions(-) delete mode 100644 db/migrate/20240228213914_add_concurrent_import_jobs_to_application_settings.rb diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index a406bbfa96210b..c4a5c167d54501 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -606,7 +606,10 @@ def self.kroki_formats_attributes jsonb_accessor :rate_limits, members_delete_limit: [:integer, { default: 60 }], - downstream_pipeline_trigger_limit_per_project_user_sha: [:integer, { default: 0 }] + downstream_pipeline_trigger_limit_per_project_user_sha: [:integer, { default: 0 }], + concurrent_github_import_jobs_limit: [:integer, { default: 1000 }], + concurrent_bitbucket_import_jobs_limit: [:integer, { default: 100 }], + concurrent_bitbucket_server_import_jobs_limit: [:integer, { default: 100 }] validates :rate_limits, json_schema: { filename: "application_setting_rate_limits" } diff --git a/app/validators/json_schemas/application_setting_rate_limits.json b/app/validators/json_schemas/application_setting_rate_limits.json index 369dbe05911429..09017429033fd6 100644 --- a/app/validators/json_schemas/application_setting_rate_limits.json +++ b/app/validators/json_schemas/application_setting_rate_limits.json @@ -4,6 +4,21 @@ "type": "object", "additionalProperties": false, "properties": { + "concurrent_bitbucket_import_jobs_limit": { + "type": "integer", + "minimum": 1, + "description": "Maximum number of simultaneous import jobs for Bitbucket Cloud importer" + }, + "concurrent_bitbucket_server_import_jobs_limit": { + "type": "integer", + "minimum": 1, + "description": "Maximum number of simultaneous import jobs for Bitbucket Server importer" + }, + "concurrent_github_import_jobs_limit": { + "type": "integer", + "minimum": 1, + "description": "Maximum number of simultaneous import jobs for GitHub importer" + }, "members_delete_limit": { "type": "integer", "minimum": 0, diff --git a/db/migrate/20240228213914_add_concurrent_import_jobs_to_application_settings.rb b/db/migrate/20240228213914_add_concurrent_import_jobs_to_application_settings.rb deleted file mode 100644 index 4491bafecdf5c0..00000000000000 --- a/db/migrate/20240228213914_add_concurrent_import_jobs_to_application_settings.rb +++ /dev/null @@ -1,13 +0,0 @@ -# frozen_string_literal: true - -class AddConcurrentImportJobsToApplicationSettings < Gitlab::Database::Migration[2.2] - milestone '16.10' - enable_lock_retries! - - def change - add_column :application_settings, :concurrent_github_import_jobs_limit, :integer, default: 1000, null: false - add_column :application_settings, :concurrent_bitbucket_import_jobs_limit, :integer, default: 100, null: false - add_column :application_settings, :concurrent_bitbucket_server_import_jobs_limit, :integer, default: 100, - null: false - end -end diff --git a/db/structure.sql b/db/structure.sql index 978cb95d0928f5..4ad48e8acdefb2 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -4120,9 +4120,6 @@ CREATE TABLE application_settings ( asciidoc_max_includes smallint DEFAULT 32 NOT NULL, clickhouse jsonb DEFAULT '{}'::jsonb NOT NULL, include_optional_metrics_in_service_ping boolean DEFAULT true NOT NULL, - concurrent_github_import_jobs_limit integer DEFAULT 1000 NOT NULL, - concurrent_bitbucket_import_jobs_limit integer DEFAULT 100 NOT NULL, - concurrent_bitbucket_server_import_jobs_limit integer DEFAULT 100 NOT NULL, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), CONSTRAINT app_settings_container_registry_pre_import_tags_rate_positive CHECK ((container_registry_pre_import_tags_rate >= (0)::numeric)), CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)), -- GitLab From a1d03f5077b60ee49064fb6316bf4250cf4eb89e Mon Sep 17 00:00:00 2001 From: Rodrigo Tomonari Date: Wed, 20 Mar 2024 21:02:41 -0300 Subject: [PATCH 09/10] Delete migration version --- db/schema_migrations/20240228213914 | 1 - 1 file changed, 1 deletion(-) delete mode 100644 db/schema_migrations/20240228213914 diff --git a/db/schema_migrations/20240228213914 b/db/schema_migrations/20240228213914 deleted file mode 100644 index 36923a2dbb8c96..00000000000000 --- a/db/schema_migrations/20240228213914 +++ /dev/null @@ -1 +0,0 @@ -ad95ee3e8837ea35e1e7ae39ab75fc609016d228a48b4abed669862b567d32c2 \ No newline at end of file -- GitLab From 3770e8a2cc44c060e30c7991f8d2069fda8f48e3 Mon Sep 17 00:00:00 2001 From: Rodrigo Tomonari Date: Thu, 21 Mar 2024 10:40:16 -0300 Subject: [PATCH 10/10] Update GitLab version in docs --- doc/api/settings.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/api/settings.md b/doc/api/settings.md index 5217a60eabb791..55acfb0129f593 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -359,9 +359,9 @@ listed in the descriptions of the relevant settings. | `check_namespace_plan` | boolean | no | Enabling this makes only licensed EE features available to projects if the project namespace's plan includes the feature or if the project is public. Premium and Ultimate only. | | `ci_max_total_yaml_size_bytes` | integer | no | The maximum amount of memory, in bytes, that can be allocated for the pipeline configuration, with all included YAML configuration files. | | `ci_max_includes` | integer | no | The [maximum number of includes](../administration/settings/continuous_integration.md#maximum-includes) per pipeline. Default is `150`. | -| `concurrent_github_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the GitHub importer. Default is 1000. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | -| `concurrent_bitbucket_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the Bitbucket Cloud importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | -| `concurrent_bitbucket_server_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the Bitbucket Server importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.10. | +| `concurrent_github_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the GitHub importer. Default is 1000. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.11. | +| `concurrent_bitbucket_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the Bitbucket Cloud importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.11. | +| `concurrent_bitbucket_server_import_jobs_limit` | integer | no | Maximum number of simultaneous import jobs for the Bitbucket Server importer. Default is 100. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/143875) in GitLab 16.11. | | `commit_email_hostname` | string | no | Custom hostname (for private commit emails). | | `container_expiration_policies_enable_historic_entries` | boolean | no | Enable [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#enable-the-cleanup-policy) for all projects. | | `container_registry_cleanup_tags_service_max_list_size` | integer | no | The maximum number of tags that can be deleted in a single execution of [cleanup policies](../user/packages/container_registry/reduce_container_registry_storage.md#set-cleanup-limits-to-conserve-resources). | -- GitLab