diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index a45425474b5e23be5fbf3372f915192503847064..ef2233307eda456de62dd4af1c6684f27526f6e3 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -238,6 +238,7 @@ def visible_attributes :container_expiration_policies_enable_historic_entries, :container_registry_expiration_policies_caching, :container_registry_token_expire_delay, + :decompress_archive_file_timeout, :default_artifacts_expire_in, :default_branch_name, :default_branch_protection, diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index f67efaf4f589d2e492dea9886eb7f12ff06c49fc..9c2101c50d4a481a45340fdf3a13cf533dc954b7 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -300,6 +300,10 @@ def self.kroki_formats_attributes presence: true, numericality: { only_integer: true, greater_than: 0 } + validates :decompress_archive_file_timeout, + presence: true, + numericality: { only_integer: true, greater_than_or_equal_to: 0 } + validates :repository_storages, presence: true validate :check_repository_storages validate :check_repository_storages_weighted diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index f6bf535158a83375b68691f5b8812429375e53f5..d24da3d4237de2ccb394cbca0dfce3b985e862c0 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -53,6 +53,7 @@ def defaults # rubocop:disable Metrics/AbcSize container_registry_vendor: '', container_registry_version: '', custom_http_clone_url_root: nil, + decompress_archive_file_timeout: 210, default_artifacts_expire_in: '30 days', default_branch_name: nil, default_branch_protection: Settings.gitlab['default_branch_protection'], diff --git a/app/views/admin/application_settings/_account_and_limit.html.haml b/app/views/admin/application_settings/_account_and_limit.html.haml index d8d6af606ac45323f7236be2903b8f87e586ff72..d518c4d5e1b323238903dc5092d9e048d3c8de6e 100644 --- a/app/views/admin/application_settings/_account_and_limit.html.haml +++ b/app/views/admin/application_settings/_account_and_limit.html.haml @@ -37,6 +37,10 @@ = f.label :max_decompressed_archive_size, s_('Import|Maximum decompressed size (MiB)'), class: 'label-light' = f.number_field :max_decompressed_archive_size, class: 'form-control gl-form-input', title: s_('Import|Maximum size of decompressed archive.'), data: { toggle: 'tooltip', container: 'body' } %span.form-text.text-muted= _('Set to 0 for no size limit.') + .form-group + = f.label :decompress_archive_file_timeout, s_('Import|Timeout for decompressing archived files (seconds)'), class: 'label-light' + = f.number_field :decompress_archive_file_timeout, class: 'form-control gl-form-input', title: s_('Import|Timeout for decompressing archived files.'), data: { toggle: 'tooltip', container: 'body' } + %span.form-text.text-muted= _('Set to 0 to disable timeout.') .form-group = f.label :session_expire_delay, _('Session duration (minutes)'), class: 'label-light' = f.number_field :session_expire_delay, class: 'form-control gl-form-input', title: _('Maximum duration of a session.'), data: { toggle: 'tooltip', container: 'body' } diff --git a/db/migrate/20230814181359_add_decompress_archive_file_timeout_to_application_setting.rb b/db/migrate/20230814181359_add_decompress_archive_file_timeout_to_application_setting.rb new file mode 100644 index 0000000000000000000000000000000000000000..f57545424a635f44cac023001b5bcb6f9dcd194a --- /dev/null +++ b/db/migrate/20230814181359_add_decompress_archive_file_timeout_to_application_setting.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class AddDecompressArchiveFileTimeoutToApplicationSetting < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def change + add_column :application_settings, :decompress_archive_file_timeout, :integer, default: 210, null: false + end +end diff --git a/db/schema_migrations/20230814181359 b/db/schema_migrations/20230814181359 new file mode 100644 index 0000000000000000000000000000000000000000..17086601248ad23d2190464912c5da5db6f89d2d --- /dev/null +++ b/db/schema_migrations/20230814181359 @@ -0,0 +1 @@ +2765c209b9347c0c76257a1af473edd6aebdc3f8d6dce3eed4d089e700c36808 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 7d9f78de7a54c90d980ac198400c4cce86070eb5..0380805513d064fbe52fc4c7c0a94865c499ff9d 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11932,6 +11932,7 @@ CREATE TABLE application_settings ( max_decompressed_archive_size integer DEFAULT 25600 NOT NULL, ci_max_total_yaml_size_bytes integer DEFAULT 157286400 NOT NULL, prometheus_alert_db_indicators_settings jsonb, + decompress_archive_file_timeout integer DEFAULT 210 NOT NULL, CONSTRAINT app_settings_container_reg_cleanup_tags_max_list_size_positive CHECK ((container_registry_cleanup_tags_service_max_list_size >= 0)), CONSTRAINT app_settings_container_registry_pre_import_tags_rate_positive CHECK ((container_registry_pre_import_tags_rate >= (0)::numeric)), CONSTRAINT app_settings_dep_proxy_ttl_policies_worker_capacity_positive CHECK ((dependency_proxy_ttl_group_policy_worker_capacity >= 0)), diff --git a/doc/administration/settings/account_and_limit_settings.md b/doc/administration/settings/account_and_limit_settings.md index 3d632880113fe15fddd8181d4299826de856ce65..e5440d4f23c20bd922eca0a03db8afdb3b67364e 100644 --- a/doc/administration/settings/account_and_limit_settings.md +++ b/doc/administration/settings/account_and_limit_settings.md @@ -155,6 +155,20 @@ To modify the maximum decompressed file size for imports in GitLab: 1. Expand **Account and limit**. 1. Set another value for **Maximum decompressed size (MiB)**. +## Timeout for decompressing archived files + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/128218) in GitLab 16.4. + +When you [import a project](../../user/project/settings/import_export.md), you can specify the maximum time out for decompressing imported archives. The default value is 210 seconds. + +To modify the maximum decompressed file size for imports in GitLab: + +1. On the left sidebar, expand the top-most chevron (**{chevron-down}**). +1. Select **Admin Area**. +1. Select **Settings > General**. +1. Expand **Account and limit**. +1. Set another value for **Timeout for decompressing archived files (seconds)**. + ## Personal access token prefix > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/20968) in GitLab 13.7. diff --git a/doc/api/settings.md b/doc/api/settings.md index f43693c26f877e4ddb866bfd32125146469b721c..6d21f0b120d4ef578c1cc6de5250ee6f255123b6 100644 --- a/doc/api/settings.md +++ b/doc/api/settings.md @@ -65,6 +65,7 @@ Example response: "container_registry_expiration_policies_caching": true, "container_registry_expiration_policies_worker_capacity": 4, "container_registry_token_expire_delay": 5, + "decompress_archive_file_timeout": 210, "repository_storages_weighted": {"default": 100}, "plantuml_enabled": false, "plantuml_url": null, @@ -208,6 +209,7 @@ Example response: "container_registry_expiration_policies_caching": true, "container_registry_expiration_policies_worker_capacity": 4, "container_registry_token_expire_delay": 5, + "decompress_archive_file_timeout": 210, "package_registry_cleanup_policies_worker_capacity": 2, "repository_storages": ["default"], "plantuml_enabled": false, @@ -346,6 +348,7 @@ listed in the descriptions of the relevant settings. | `allow_account_deletion` | boolean | no | Enable [users to delete their accounts](../administration/settings/account_and_limit_settings.md#prevent-users-from-deleting-their-accounts). | | `deactivate_dormant_users` | boolean | no | Enable [automatic deactivation of dormant users](../administration/moderate_users.md#automatically-deactivate-dormant-users). | | `deactivate_dormant_users_period` | integer | no | Length of time (in days) after which a user is considered dormant. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/336747) in GitLab 15.3. | +| `decompress_archive_file_timeout` | integer | no | Default timeout for decompressing archived files, in seconds. Set to 0 to disable timeouts. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/129161) in GitLab 16.4. | | `default_artifacts_expire_in` | string | no | Set the default expiration time for each job's artifacts. | | `default_branch_name` | string | no | [Instance-level custom initial branch name](../user/project/repository/branches/default.md#instance-level-custom-initial-branch-name). [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/225258) in GitLab 13.2. | | `default_branch_protection` | integer | no | Determine if developers can push to the default branch. Can take: `0` _(not protected, both users with the Developer role or Maintainer role can push new commits and force push)_, `1` _(partially protected, users with the Developer role or Maintainer role can push new commits, but cannot force push)_ or `2` _(fully protected, users with the Developer or Maintainer role cannot push new commits, but users with the Developer or Maintainer role can; no one can force push)_ as a parameter. Default is `2`. | diff --git a/lib/api/settings.rb b/lib/api/settings.rb index e2dc78fe84a8188f077ecaf0a8220ad687bc6b0c..61c6eb7d02a26adbd377175d3466e0fc92354977 100644 --- a/lib/api/settings.rb +++ b/lib/api/settings.rb @@ -45,6 +45,7 @@ def filter_attributes_using_license(attrs) optional :asset_proxy_whitelist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Deprecated: Use :asset_proxy_allowlist instead. Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically whitelisted.' optional :asset_proxy_allowlist, type: Array[String], coerce_with: Validations::Types::CommaSeparatedToArray.coerce, desc: 'Assets that match these domain(s) will NOT be proxied. Wildcards allowed. Your GitLab installation URL is automatically allowed.' optional :container_registry_token_expire_delay, type: Integer, desc: 'Authorization token duration (minutes)' + optional :decompress_archive_file_timeout, type: Integer, desc: 'Default timeout for decompressing archived files, in seconds. Set to 0 to disable timeouts.' optional :default_artifacts_expire_in, type: String, desc: "Set the default expiration time for each job's artifacts" optional :default_ci_config_path, type: String, desc: 'The instance default CI/CD configuration file and path for new projects' optional :default_project_creation, type: Integer, values: ::Gitlab::Access.project_creation_values, desc: 'Determine if developers can create projects in the group' diff --git a/lib/gitlab/import_export/decompressed_archive_size_validator.rb b/lib/gitlab/import_export/decompressed_archive_size_validator.rb index 3609df89958bcf0fbe1672664a3267a616a7dfca..23a0c9a1f71056ce5c39f75dc44a3b2b9c2d13e9 100644 --- a/lib/gitlab/import_export/decompressed_archive_size_validator.rb +++ b/lib/gitlab/import_export/decompressed_archive_size_validator.rb @@ -5,13 +5,12 @@ module ImportExport class DecompressedArchiveSizeValidator include Gitlab::Utils::StrongMemoize - TIMEOUT_LIMIT = 210.seconds - ServiceError = Class.new(StandardError) - def initialize(archive_path:, max_bytes: self.class.max_bytes) + def initialize(archive_path:, max_bytes: self.class.max_bytes, timeout: self.class.timeout) @archive_path = archive_path @max_bytes = max_bytes + @timeout = timeout end def valid? @@ -24,6 +23,10 @@ def self.max_bytes Gitlab::CurrentSettings.current_application_settings.max_decompressed_archive_size.megabytes end + def self.timeout + Gitlab::CurrentSettings.current_application_settings.decompress_archive_file_timeout + end + private def validate @@ -32,7 +35,7 @@ def validate validate_archive_path - Timeout.timeout(TIMEOUT_LIMIT) do + Timeout.timeout(@timeout) do stderr_r, stderr_w = IO.pipe stdout, wait_threads = Open3.pipeline_r(*command, pgroup: true, err: stderr_w) @@ -70,7 +73,7 @@ def validate valid_archive rescue Timeout::Error - log_error('Timeout reached during archive decompression') + log_error("Timeout of #{@timeout} seconds reached during archive decompression") pgrps.each { |pgrp| Process.kill(-1, pgrp) } if pgrps diff --git a/locale/gitlab.pot b/locale/gitlab.pot index aa40012f77afb4c9f316e7553a859663c9e8e587..73eee05e89b25ea6deabd561014d231550bb9ec8 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -24241,6 +24241,12 @@ msgstr "" msgid "Import|There is not a valid Git repository at this URL. If your HTTP repository is not publicly accessible, verify your credentials." msgstr "" +msgid "Import|Timeout for decompressing archived files (seconds)" +msgstr "" + +msgid "Import|Timeout for decompressing archived files." +msgstr "" + msgid "Improve customer support with Service Desk" msgstr "" @@ -43816,6 +43822,9 @@ msgstr "" msgid "Set to 0 for no size limit." msgstr "" +msgid "Set to 0 to disable timeout." +msgstr "" + msgid "Set to auto-merge" msgstr "" diff --git a/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb b/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb index aceea70be921388e7f96e4624f2d0766768d6721..a73f5d44b50177fad41d159376db87c288a71337 100644 --- a/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb +++ b/spec/lib/gitlab/import_export/decompressed_archive_size_validator_spec.rb @@ -93,7 +93,7 @@ end context 'when timeout occurs' do - let(:error_message) { 'Timeout reached during archive decompression' } + let(:error_message) { 'Timeout of 210 seconds reached during archive decompression' } let(:exception) { Timeout::Error } include_examples 'logs raised exception and terminates validator process group' diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 5a70bec8b33aba5d6005f0effad6f35b4f14cc34..6214e5f60f91bd13d36b7a3340f9b649cf6793d6 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -25,6 +25,7 @@ it { expect(setting.kroki_formats).to eq({}) } it { expect(setting.default_branch_protection_defaults).to eq({}) } it { expect(setting.max_decompressed_archive_size).to eq(25600) } + it { expect(setting.decompress_archive_file_timeout).to eq(210) } end describe 'validations' do @@ -134,6 +135,9 @@ it { is_expected.to validate_presence_of(:container_registry_import_target_plan) } it { is_expected.to validate_presence_of(:container_registry_import_created_before) } + it { is_expected.to validate_numericality_of(:decompress_archive_file_timeout).only_integer.is_greater_than_or_equal_to(0) } + it { is_expected.not_to allow_value(nil).for(:decompress_archive_file_timeout) } + it { is_expected.to validate_numericality_of(:dependency_proxy_ttl_group_policy_worker_capacity).only_integer.is_greater_than_or_equal_to(0) } it { is_expected.not_to allow_value(nil).for(:dependency_proxy_ttl_group_policy_worker_capacity) } diff --git a/spec/requests/api/settings_spec.rb b/spec/requests/api/settings_spec.rb index 12af1fc1b7993c3a5e38212fb395c7b3970543b8..7d630f1868155cc38e41c9062b73c5d4e626f6d9 100644 --- a/spec/requests/api/settings_spec.rb +++ b/spec/requests/api/settings_spec.rb @@ -27,6 +27,7 @@ expect(json_response['secret_detection_token_revocation_url']).to be_nil expect(json_response['secret_detection_revocation_token_types_url']).to be_nil expect(json_response['sourcegraph_public_only']).to be_truthy + expect(json_response['decompress_archive_file_timeout']).to eq(210) expect(json_response['default_preferred_language']).to be_a String expect(json_response['default_project_visibility']).to be_a String expect(json_response['default_snippet_visibility']).to be_a String @@ -153,6 +154,7 @@ enforce_terms: true, terms: 'Hello world!', performance_bar_allowed_group_path: group.full_path, + decompress_archive_file_timeout: 60, diff_max_patch_bytes: 300_000, diff_max_files: 2000, diff_max_lines: 50000, @@ -234,6 +236,7 @@ expect(json_response['enforce_terms']).to be(true) expect(json_response['terms']).to eq('Hello world!') expect(json_response['performance_bar_allowed_group_id']).to eq(group.id) + expect(json_response['decompress_archive_file_timeout']).to eq(60) expect(json_response['diff_max_patch_bytes']).to eq(300_000) expect(json_response['diff_max_files']).to eq(2000) expect(json_response['diff_max_lines']).to eq(50000)