diff --git a/app/services/projects/import_service.rb b/app/services/projects/import_service.rb index ceab7098b3269314a3703710c0460d650fd6b47f..e22b728cea30f3276a496176db4ea89100a1ede3 100644 --- a/app/services/projects/import_service.rb +++ b/app/services/projects/import_service.rb @@ -20,6 +20,8 @@ def execute add_repository_to_project + validate_repository_size! + download_lfs_objects import_data @@ -58,6 +60,10 @@ def extra_attributes_for_measurement attr_reader :resolved_address + def validate_repository_size! + # Defined in EE::Projects::ImportService + end + def after_execute_hook # Defined in EE::Projects::ImportService end diff --git a/config/feature_flags/development/post_import_repository_size_check.yml b/config/feature_flags/development/post_import_repository_size_check.yml new file mode 100644 index 0000000000000000000000000000000000000000..a51e9085b9b6b9a3a2771e6a75cf6f513034d695 --- /dev/null +++ b/config/feature_flags/development/post_import_repository_size_check.yml @@ -0,0 +1,8 @@ +--- +name: post_import_repository_size_check +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/122814 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/414530 +milestone: '16.1' +type: development +group: group::import and integrate +default_enabled: false diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 5c6ec888e54342be78574d1189ab9d5516e007c8..a6be4de237217405d2b7c56e9eaa2cba641fe1ae 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -491,6 +491,8 @@ - 1 - - projects_register_suggested_reviewers_project - 1 +- - projects_repository_destroy + - 1 - - projects_schedule_bulk_repository_shard_moves - 1 - - projects_update_repository_storage diff --git a/ee/app/services/ee/projects/import_service.rb b/ee/app/services/ee/projects/import_service.rb index 2fcbf70e4c94162f40382e4d90049b2c1a23e1f7..5b3350ea9c4679fe32c96cac315cf741d500ca8d 100644 --- a/ee/app/services/ee/projects/import_service.rb +++ b/ee/app/services/ee/projects/import_service.rb @@ -5,6 +5,11 @@ module Projects module ImportService extend ::Gitlab::Utils::Override + override :validate_repository_size! + def validate_repository_size! + ::Import::ValidateRepositorySizeService.new(project).execute + end + override :after_execute_hook def after_execute_hook super diff --git a/ee/app/services/import/validate_repository_size_service.rb b/ee/app/services/import/validate_repository_size_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..dacd2667b0fb9ba738f3a9c4da02666ed76b6b95 --- /dev/null +++ b/ee/app/services/import/validate_repository_size_service.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Import + class ValidateRepositorySizeService + def initialize(project) + @project = project + end + + def execute + return unless ::Feature.enabled?(:post_import_repository_size_check) + + project.repository.expire_content_cache + + ::Projects::UpdateStatisticsService.new(project, nil, statistics: [:repository_size]).execute + + return unless project.repository_size_checker.above_size_limit? + + ::Projects::RepositoryDestroyWorker.perform_async(project.id) + + raise ::Projects::ImportService::Error, s_("ImportProjects|Repository above permitted size limit.") + end + + private + + attr_reader :project + end +end diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml index b4bf0adc6696fa704388d777d5658b55d2a65abd..926a4f817755a1f48a43c61819d605a2950fe80b 100644 --- a/ee/app/workers/all_queues.yml +++ b/ee/app/workers/all_queues.yml @@ -1623,6 +1623,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: projects_repository_destroy + :worker_name: Projects::RepositoryDestroyWorker + :feature_category: :importers + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: pull_mirrors_reenable_configuration :worker_name: PullMirrors::ReenableConfigurationWorker :feature_category: :source_code_management diff --git a/ee/app/workers/projects/repository_destroy_worker.rb b/ee/app/workers/projects/repository_destroy_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..edbcae0da5d2742d7b42b474a84b7053e200e794 --- /dev/null +++ b/ee/app/workers/projects/repository_destroy_worker.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Projects + class RepositoryDestroyWorker + include ApplicationWorker + + data_consistency :delayed + feature_category :importers + idempotent! + + def perform(project_id) + project = ::Project.find_by_id(project_id) + return unless project + + ::Repositories::DestroyService.new(project.repository).execute + + # Because the repository is destroyed inside a run_after_commit callback, we need to trigger the callback + project.touch + end + end +end diff --git a/ee/lib/ee/gitlab/bitbucket_server_import/importers/repository_importer.rb b/ee/lib/ee/gitlab/bitbucket_server_import/importers/repository_importer.rb new file mode 100644 index 0000000000000000000000000000000000000000..759cc41bf25c560dfb754385462abc53295aa4f1 --- /dev/null +++ b/ee/lib/ee/gitlab/bitbucket_server_import/importers/repository_importer.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module EE + module Gitlab + module BitbucketServerImport + module Importers + module RepositoryImporter + extend ::Gitlab::Utils::Override + + override :validate_repository_size! + def validate_repository_size! + ::Import::ValidateRepositorySizeService.new(project).execute + end + end + end + end + end +end diff --git a/ee/lib/ee/gitlab/github_import/importer/repository_importer.rb b/ee/lib/ee/gitlab/github_import/importer/repository_importer.rb new file mode 100644 index 0000000000000000000000000000000000000000..2890e3c0fc928469fdd216d9c7a2e8359f6f6bfb --- /dev/null +++ b/ee/lib/ee/gitlab/github_import/importer/repository_importer.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module EE + module Gitlab + module GithubImport + module Importer + module RepositoryImporter + extend ::Gitlab::Utils::Override + + override :validate_repository_size! + def validate_repository_size! + ::Import::ValidateRepositorySizeService.new(project).execute + end + end + end + end + end +end diff --git a/ee/spec/lib/ee/gitlab/bitbucket_server_import/importers/repository_importer_spec.rb b/ee/spec/lib/ee/gitlab/bitbucket_server_import/importers/repository_importer_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..153e6afa04e1bbf06c6f94bc693a8e8b7efd775d --- /dev/null +++ b/ee/spec/lib/ee/gitlab/bitbucket_server_import/importers/repository_importer_spec.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BitbucketServerImport::Importers::RepositoryImporter, feature_category: :importers do + let_it_be(:project) { create(:project, import_url: 'http://bitbucket:test@my-bitbucket') } + + subject(:importer) { described_class.new(project) } + + describe '#execute' do + it 'validates repository size' do + allow(project.repository).to receive(:import_repository) + allow(project.repository).to receive(:fetch_as_mirror) + + expect_next_instance_of(::Import::ValidateRepositorySizeService, project) do |service| + expect(service).to receive(:execute) + end + + importer.execute + end + end +end diff --git a/ee/spec/lib/ee/gitlab/github_import/importer/repository_importer_spec.rb b/ee/spec/lib/ee/gitlab/github_import/importer/repository_importer_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..94ae63cb65d66d329999110eb9d64f2d46ea92a3 --- /dev/null +++ b/ee/spec/lib/ee/gitlab/github_import/importer/repository_importer_spec.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter, feature_category: :importers do + let(:project) { build_stubbed(:project) } + let(:client) { instance_double(Gitlab::GithubImport::Client) } + + subject(:importer) { described_class.new(project, client) } + + describe '#import_repository' do + it 'validates repository size' do + allow(project).to receive(:ensure_repository) + allow(project).to receive_message_chain(:repository, :fetch_as_mirror) + allow(client).to receive(:repository).and_return({}) + + expect_next_instance_of(::Import::ValidateRepositorySizeService, project) do |service| + expect(service).to receive(:execute) + end + + importer.import_repository + end + end +end diff --git a/ee/spec/services/import/validate_repository_size_service_spec.rb b/ee/spec/services/import/validate_repository_size_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..c89472f5063bc5b3a22bd7d694724c8eeebbebb2 --- /dev/null +++ b/ee/spec/services/import/validate_repository_size_service_spec.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Import::ValidateRepositorySizeService, feature_category: :importers do + let_it_be(:project) { create(:project) } + + let(:above_size_limit) { true } + + subject(:service) { described_class.new(project) } + + describe '#execute' do + before do + allow_next_instance_of(Gitlab::RepositorySizeChecker) do |checker| + allow(checker).to receive(:above_size_limit?).and_return(above_size_limit) + end + end + + context 'when `post_import_repository_size_check` feature flag is enabled' do + context 'when repository size is over the limit' do + let(:above_size_limit) { true } + + it 'schedules worker to destroy repository and raises error' do + expect(::Projects::RepositoryDestroyWorker).to receive(:perform_async).with(project.id) + + expect { service.execute } + .to raise_error(::Projects::ImportService::Error, 'Repository above permitted size limit.') + end + end + + context 'when repository size is not over the limit' do + let(:above_size_limit) { false } + + it 'does nothing' do + expect(::Projects::RepositoryDestroyWorker).not_to receive(:perform_async) + + expect(service.execute).to eq(nil) + end + end + end + + context 'when `post_import_repository_size_check` feature flag is disabled' do + it 'does nothing' do + expect(::Projects::RepositoryDestroyWorker).not_to receive(:perform_async) + + expect(service.execute).to eq(nil) + end + end + end +end diff --git a/ee/spec/services/projects/import_service_spec.rb b/ee/spec/services/projects/import_service_spec.rb index 0c74d89d98edfd9281abfc5476a0485438758870..1ef412b6a531adb3186ee13059b85dc17a91006a 100644 --- a/ee/spec/services/projects/import_service_spec.rb +++ b/ee/spec/services/projects/import_service_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Projects::ImportService do +RSpec.describe Projects::ImportService, feature_category: :importers do let_it_be(:project) { create(:project) } let_it_be(:user) { project.creator } @@ -66,4 +66,12 @@ end end end + + it 'validates repository size' do + expect_next_instance_of(::Import::ValidateRepositorySizeService, project) do |service| + expect(service).to receive(:execute) + end + + subject.execute + end end diff --git a/ee/spec/workers/projects/repository_destroy_worker_spec.rb b/ee/spec/workers/projects/repository_destroy_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..6a980ccba874dbb68ded641f0a18275f04aefba8 --- /dev/null +++ b/ee/spec/workers/projects/repository_destroy_worker_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::RepositoryDestroyWorker, feature_category: :importers do + let_it_be(:project) { create(:project) } + + subject(:worker) { described_class.new } + + describe '#perform' do + it_behaves_like 'an idempotent worker' do + let(:job_args) { project.id } + end + + it 'destroy repository' do + expect_next_instance_of(::Repositories::DestroyService) do |service| + expect(service).to receive(:execute) + end + + worker.perform(project.id) + end + + context 'when project does not exist' do + it 'does not destroy repository' do + expect(::Repositories::DestroyService).not_to receive(:new) + + worker.perform(non_existing_record_id) + end + end + end +end diff --git a/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb b/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb index cd09ac40e9f77e64bff1907647df5f1b0ff6181a..e7a9adf2beb78b38a67e51a80aa3ed1d76365dac 100644 --- a/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb +++ b/lib/gitlab/bitbucket_server_import/importers/repository_importer.rb @@ -17,6 +17,8 @@ def execute project.repository.import_repository(project.import_url) project.repository.fetch_as_mirror(project.import_url, refmap: refmap) + validate_repository_size! + update_clone_time end @@ -48,7 +50,13 @@ def refmap def update_clone_time project.touch(:last_repository_updated_at) end + + def validate_repository_size! + # Defined in EE + end end end end end + +Gitlab::BitbucketServerImport::Importers::RepositoryImporter.prepend_mod diff --git a/lib/gitlab/github_import/importer/repository_importer.rb b/lib/gitlab/github_import/importer/repository_importer.rb index 2654812b64ab883323fade40f56d9a6be14c6909..d37942aa8a3c016be4a2a9dc1295e3ead50cc21e 100644 --- a/lib/gitlab/github_import/importer/repository_importer.rb +++ b/lib/gitlab/github_import/importer/repository_importer.rb @@ -54,6 +54,8 @@ def import_repository project.change_head(default_branch) if default_branch + validate_repository_size! + # The initial fetch can bring in lots of loose refs and objects. # Running a `git gc` will make importing pull requests faster. Repositories::HousekeepingService.new(project, :gc).execute @@ -89,7 +91,13 @@ def default_branch strong_memoize_attr def client_repository client.repository(project.import_source) end + + def validate_repository_size! + # Defined in EE + end end end end end + +Gitlab::GithubImport::Importer::RepositoryImporter.prepend_mod diff --git a/locale/gitlab.pot b/locale/gitlab.pot index ef2498be7b2f500bdb1f1ddd5c445be494374ba8..ad029c7f10b27f106c6e161453bc65b7eaa4032c 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -23144,6 +23144,9 @@ msgstr "" msgid "ImportProjects|Re-import creates a new project. It does not sync with the existing project." msgstr "" +msgid "ImportProjects|Repository above permitted size limit." +msgstr "" + msgid "ImportProjects|Requesting namespaces failed" msgstr "" diff --git a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb index 0b8b1922d94fcbe8ad5fce25098a9ff2f4d86a6b..6b3d4485ea57c99e1de048b055f3be1c0586e0b3 100644 --- a/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb +++ b/spec/lib/gitlab/github_import/importer/repository_importer_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter do +RSpec.describe Gitlab::GithubImport::Importer::RepositoryImporter, feature_category: :importers do let(:repository) { double(:repository) } let(:import_state) { double(:import_state) } let(:client) { double(:client) } @@ -23,6 +23,7 @@ let(:project) do double( :project, + id: 1, import_url: 'foo.git', import_source: 'foo/bar', repository_storage: 'foo', @@ -204,6 +205,8 @@ .to receive(:fetch_as_mirror) .with(project.import_url, refmap: Gitlab::GithubImport.refmap, forced: true) + expect(importer).to receive(:validate_repository_size!) + service = double expect(Repositories::HousekeepingService) .to receive(:new).with(project, :gc).and_return(service)