diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index bf68101934b2a6385b34b64a3960b1dc6ba12d1a..435e046f3f7ea859787c24529ff0e84dba904a30 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -645,6 +645,7 @@ def self.kroki_formats_attributes reset_memoized_terms end after_commit :expire_performance_bar_allowed_user_ids_cache, if: -> { previous_changes.key?('performance_bar_allowed_group_id') } + after_commit :reset_deletion_warning_redis_key, if: :saved_change_to_inactive_projects_delete_after_months? def validate_grafana_url validate_url(parsed_grafana_url, :grafana_url, GRAFANA_URL_ERROR_MESSAGE) @@ -775,6 +776,10 @@ def validate_url(parsed_url, name, error_message) ) end end + + def reset_deletion_warning_redis_key + Gitlab::InactiveProjectsDeletionWarningTracker.reset_all + end end ApplicationSetting.prepend_mod_with('ApplicationSetting') diff --git a/app/models/event.rb b/app/models/event.rb index e9a98c06b59685381b18c8993012e01f691dbb7d..f53d56e735e40b7416f6586cfff93af8b4d5d82c 100644 --- a/app/models/event.rb +++ b/app/models/event.rb @@ -357,6 +357,8 @@ def reset_project_activity Project.unscoped.where(id: project_id) .where('last_activity_at <= ?', RESET_PROJECT_ACTIVITY_INTERVAL.ago) .touch_all(:last_activity_at, time: created_at) # rubocop: disable Rails/SkipsModelValidations + + Gitlab::InactiveProjectsDeletionWarningTracker.new(project.id).reset end def authored_by?(user) diff --git a/app/models/project.rb b/app/models/project.rb index 027b2257f2e13b43cbdfed8ff20077deb870ff22..f3d8c3dfced420083f51160b2032e022344bdc5f 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -745,6 +745,16 @@ def self.wrap_with_cte(collection) Project.with(cte.to_arel).from(cte.alias_to(Project.arel_table)) end + def self.inactive + project_statistics = ::ProjectStatistics.arel_table + minimum_size_mb = ::Gitlab::CurrentSettings.inactive_projects_min_size_mb.megabytes + last_activity_cutoff = ::Gitlab::CurrentSettings.inactive_projects_send_warning_email_after_months.months.ago + + joins(:statistics) + .where((project_statistics[:storage_size]).gt(minimum_size_mb)) + .where('last_activity_at < ?', last_activity_cutoff) + end + scope :active, -> { joins(:issues, :notes, :merge_requests).order('issues.created_at, notes.created_at, merge_requests.created_at DESC') } scope :abandoned, -> { where('projects.last_activity_at < ?', 6.months.ago) } diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index c436335a24484f1164c9ff11ba5bbd69d3f90393..63e5210936bec36254f7cad6b1f9586fde226b40 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -579,6 +579,15 @@ :weight: 1 :idempotent: :tags: [] +- :name: cronjob:projects_inactive_projects_deletion_cron + :worker_name: Projects::InactiveProjectsDeletionCronWorker + :feature_category: :compliance_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:projects_schedule_refresh_build_artifacts_size_statistics :worker_name: Projects::ScheduleRefreshBuildArtifactsSizeStatisticsWorker :feature_category: :build_artifacts @@ -2794,6 +2803,15 @@ :weight: 1 :idempotent: :tags: [] +- :name: projects_inactive_projects_deletion_notification + :worker_name: Projects::InactiveProjectsDeletionNotificationWorker + :feature_category: :compliance_management + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: projects_post_creation :worker_name: Projects::PostCreationWorker :feature_category: :source_code_management diff --git a/app/workers/projects/inactive_projects_deletion_cron_worker.rb b/app/workers/projects/inactive_projects_deletion_cron_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..2c3f4191502d822b3d58d3f488af474f2c688ecd --- /dev/null +++ b/app/workers/projects/inactive_projects_deletion_cron_worker.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +module Projects + class InactiveProjectsDeletionCronWorker + include ApplicationWorker + include Gitlab::Utils::StrongMemoize + include CronjobQueue + + idempotent! + data_consistency :always + feature_category :compliance_management + + INTERVAL = 2.seconds.to_i + + def perform + return unless ::Gitlab::CurrentSettings.delete_inactive_projects? + + admin_user = User.admins.active.first + + return unless admin_user + + notified_inactive_projects = Gitlab::InactiveProjectsDeletionWarningTracker.notified_projects + + Project.inactive.without_deleted.find_each(batch_size: 100).with_index do |project, index| # rubocop: disable CodeReuse/ActiveRecord + next unless Feature.enabled?(:inactive_projects_deletion, project.root_namespace) + + delay = index * INTERVAL + + with_context(project: project, user: admin_user) do + deletion_warning_email_sent_on = notified_inactive_projects["project:#{project.id}"] + + if send_deletion_warning_email?(deletion_warning_email_sent_on, project) + send_notification(delay, project, admin_user) + elsif deletion_warning_email_sent_on && delete_due_to_inactivity?(deletion_warning_email_sent_on) + Gitlab::InactiveProjectsDeletionWarningTracker.new(project.id).reset + delete_project(project, admin_user) + end + end + end + end + + private + + def grace_months_after_deletion_notification + strong_memoize(:grace_months_after_deletion_notification) do + (::Gitlab::CurrentSettings.inactive_projects_delete_after_months - + ::Gitlab::CurrentSettings.inactive_projects_send_warning_email_after_months).months + end + end + + def send_deletion_warning_email?(deletion_warning_email_sent_on, project) + deletion_warning_email_sent_on.blank? + end + + def delete_due_to_inactivity?(deletion_warning_email_sent_on) + deletion_warning_email_sent_on < grace_months_after_deletion_notification.ago + end + + def deletion_date + grace_months_after_deletion_notification.from_now.to_date.to_s + end + + def delete_project(project, user) + ::Projects::DestroyService.new(project, user, {}).async_execute + end + + def send_notification(delay, project, user) + ::Projects::InactiveProjectsDeletionNotificationWorker.perform_in(delay, project.id, deletion_date) + end + end +end + +Projects::InactiveProjectsDeletionCronWorker.prepend_mod diff --git a/app/workers/projects/inactive_projects_deletion_notification_worker.rb b/app/workers/projects/inactive_projects_deletion_notification_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..0bf808fd753be824ed34868bcc296d1f77ec8f6a --- /dev/null +++ b/app/workers/projects/inactive_projects_deletion_notification_worker.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module Projects + class InactiveProjectsDeletionNotificationWorker + include ApplicationWorker + include ExceptionBacktrace + + idempotent! + data_consistency :sticky + sidekiq_options retry: 3 + feature_category :compliance_management + + def perform(project_id, deletion_date) + return if Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).notified? + + project = Project.find(project_id) + + notification_service.inactive_project_deletion_warning(project, deletion_date) + + Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).mark_notified + rescue ActiveRecord::RecordNotFound => error + Gitlab::ErrorTracking.log_exception(error, project_id: project_id) + end + + private + + def notification_service + @notification_service ||= NotificationService.new + end + end +end diff --git a/config/feature_flags/development/inactive_projects_deletion.yml b/config/feature_flags/development/inactive_projects_deletion.yml new file mode 100644 index 0000000000000000000000000000000000000000..e9bb91f62cc1742bbb585491ff6d709173e19fe0 --- /dev/null +++ b/config/feature_flags/development/inactive_projects_deletion.yml @@ -0,0 +1,8 @@ +--- +name: inactive_projects_deletion +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/357968 +milestone: '15.0' +type: development +group: group::compliance +default_enabled: false diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 24547c78d9e4d9ff6b8bdc3933b40864e7602281..5b15b119b3aeb36bc6418719d019dd9fc10b27da 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -626,6 +626,9 @@ Settings.cron_jobs['projects_schedule_refresh_build_artifacts_size_statistics_worker'] ||= Settingslogic.new({}) Settings.cron_jobs['projects_schedule_refresh_build_artifacts_size_statistics_worker']['cron'] ||= '2/17 * * * *' Settings.cron_jobs['projects_schedule_refresh_build_artifacts_size_statistics_worker']['job_class'] = 'Projects::ScheduleRefreshBuildArtifactsSizeStatisticsWorker' +Settings.cron_jobs['inactive_projects_deletion_cron_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['inactive_projects_deletion_cron_worker']['cron'] ||= '0 1 * * *' +Settings.cron_jobs['inactive_projects_deletion_cron_worker']['job_class'] = 'Projects::InactiveProjectsDeletionCronWorker' Gitlab.ee do Settings.cron_jobs['analytics_devops_adoption_create_all_snapshots_worker'] ||= Settingslogic.new({}) diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index f3df76080522256af2ed2e697442b613f4cc7bed..ddaaaf05dbdd6acdece77bfdfed673a7f7988a1d 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -359,6 +359,8 @@ - 1 - - projects_git_garbage_collect - 1 +- - projects_inactive_projects_deletion_notification + - 1 - - projects_post_creation - 1 - - projects_process_sync_events diff --git a/ee/app/models/ee/project.rb b/ee/app/models/ee/project.rb index eaf53d395fd952683ec3ea798a0a80570b2480ad..6f12b7e69965dcee6a01cb0bc33e4b393d9202f5 100644 --- a/ee/app/models/ee/project.rb +++ b/ee/app/models/ee/project.rb @@ -308,6 +308,20 @@ def with_web_entity_associations def with_api_entity_associations super.preload(group: [:ip_restrictions, :saml_provider]) end + + override :inactive + def inactive + return super unless ::Gitlab.com? + + statistics = ::ProjectStatistics.arel_table + minimum_size_mb = ::Gitlab::CurrentSettings.inactive_projects_min_size_mb.megabytes + last_activity_cutoff = ::Gitlab::CurrentSettings.inactive_projects_send_warning_email_after_months.months.ago + + for_plan_name(::Plan.default_plans) + .joins(:statistics) + .where((statistics[:storage_size]).gt(minimum_size_mb)) + .where('last_activity_at < ?', last_activity_cutoff) + end end def can_store_security_reports? diff --git a/ee/app/workers/ee/projects/inactive_projects_deletion_cron_worker.rb b/ee/app/workers/ee/projects/inactive_projects_deletion_cron_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..89a68df4ce5f1c1b6f6a7303d32e84492e43f29f --- /dev/null +++ b/ee/app/workers/ee/projects/inactive_projects_deletion_cron_worker.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +module EE + module Projects + module InactiveProjectsDeletionCronWorker + extend ::Gitlab::Utils::Override + + override :delete_project + def delete_project(project, user) + return super unless License.feature_available?(:adjourned_deletion_for_projects_and_groups) + # Can't use `project.adjourned_deletion?` see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689#note_943072034 + return super unless project.adjourned_deletion_configured? + + ::Projects::MarkForDeletionService.new(project, user, {}).execute + end + + override :send_deletion_warning_email? + def send_deletion_warning_email?(deletion_warning_email_sent_on, project) + # Can't use `project.marked_for_deletion?`, see https://gitlab.com/gitlab-org/gitlab/-/merge_requests/85689#note_943072064 + return false if project.marked_for_deletion_at? + + super + end + + override :send_notification + def send_notification(delay, project, user) + super + + ::AuditEventService.new( + user, + project, + action: :custom, + custom_message: "Project is scheduled to be deleted on #{deletion_date} due to inactivity." + ).for_project.security_event + end + end + end +end diff --git a/ee/spec/models/project_spec.rb b/ee/spec/models/project_spec.rb index 99b707f9f8764760c8a9b98ef65102887eb71e11..6038c907ef71becab79196204cd30d5a0923b935 100644 --- a/ee/spec/models/project_spec.rb +++ b/ee/spec/models/project_spec.rb @@ -5,6 +5,7 @@ RSpec.describe Project do include ProjectForksHelper include ::EE::GeoHelpers + include ::ProjectHelpers using RSpec::Parameterized::TableSyntax let(:project) { create(:project) } @@ -3365,4 +3366,57 @@ def stub_default_url_options(host) it_behaves_like 'returns true if project is inactive' end end + + describe '.inactive', :saas do + before do + stub_application_setting(inactive_projects_min_size_mb: 5) + stub_application_setting(inactive_projects_send_warning_email_after_months: 24) + end + + it 'returns inactive projects belonging to free namespace' do + ultimate_group = create(:group_with_plan, plan: :ultimate_plan) + premium_group = create(:group_with_plan, plan: :premium_plan) + free_plan_group = create(:group_with_plan, plan: :free_plan) + + free_small_active_project = + create_project_with_statistics(free_plan_group, with_data: true, size_multiplier: 1.kilobyte).tap do |project| + project.update!(last_activity_at: 7.days.ago) + end + + free_small_inactive_project = + create_project_with_statistics(free_plan_group, with_data: true, size_multiplier: 1.kilobyte).tap do |project| + project.update!(last_activity_at: 3.years.ago) + end + + free_large_inactive_project = + create_project_with_statistics(free_plan_group, with_data: true, size_multiplier: 10.megabytes).tap do |project| + project.update!(last_activity_at: 3.years.ago) + end + + free_large_active_project = + create_project_with_statistics(free_plan_group, with_data: true, size_multiplier: 10.megabytes).tap do |project| + project.update!(last_activity_at: 7.days.ago) + end + + paid_small_active_project = + create_project_with_statistics(premium_group, with_data: true, size_multiplier: 1.megabyte).tap do |project| + project.update!(last_activity_at: 7.days.ago) + end + + paid_small_inactive_project = + create_project_with_statistics(premium_group, with_data: true, size_multiplier: 1.megabyte).tap do |project| + project.update!(last_activity_at: 7.years.ago) + end + + paid_large_inactive_project = + create_project_with_statistics(ultimate_group, with_data: true, size_multiplier: 1.gigabyte).tap do |project| + project.update!(last_activity_at: 7.years.ago) + end + + expect(described_class.inactive).to contain_exactly(free_large_inactive_project) + expect(described_class.inactive) + .not_to include(free_small_active_project, free_small_inactive_project, free_large_active_project, + paid_small_active_project, paid_small_inactive_project, paid_large_inactive_project) + end + end end diff --git a/ee/spec/workers/ee/projects/inactive_projects_deletion_cron_worker_spec.rb b/ee/spec/workers/ee/projects/inactive_projects_deletion_cron_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..5ad1e91045c895e06d01b782e4cf716f9bb29934 --- /dev/null +++ b/ee/spec/workers/ee/projects/inactive_projects_deletion_cron_worker_spec.rb @@ -0,0 +1,167 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::InactiveProjectsDeletionCronWorker do + include ProjectHelpers + + describe "#perform", :clean_gitlab_redis_shared_state, :sidekiq_inline do + subject(:worker) { described_class.new } + + let_it_be(:admin_user) { create(:user, :admin) } + let_it_be(:non_admin_user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:new_blank_project) do + create_project_with_statistics.tap do |project| + project.update!(last_activity_at: Time.current) + end + end + + let_it_be(:inactive_blank_project) do + create_project_with_statistics.tap do |project| + project.update!(last_activity_at: 13.months.ago) + end + end + + let_it_be(:inactive_large_project) do + create_project_with_statistics(group, with_data: true, size_multiplier: 2.gigabytes) + .tap { |project| project.update!(last_activity_at: 2.years.ago) } + end + + let_it_be(:active_large_project) do + create_project_with_statistics(group, with_data: true, size_multiplier: 2.gigabytes) + .tap { |project| project.update!(last_activity_at: 1.month.ago) } + end + + let_it_be(:delay) { anything } + + before do + stub_application_setting(inactive_projects_min_size_mb: 5) + stub_application_setting(inactive_projects_send_warning_email_after_months: 12) + stub_application_setting(inactive_projects_delete_after_months: 14) + stub_application_setting(deletion_adjourned_period: 7) + stub_application_setting(delete_inactive_projects: true) + stub_feature_flags(inactive_projects_deletion: true) + end + + it 'does not send deletion warning email for inactive projects that are already marked for deletion' do + inactive_large_project.update!(marked_for_deletion_at: Date.current) + + expect(::Projects::InactiveProjectsDeletionNotificationWorker).not_to receive(:perform_in) + expect(::Projects::DestroyService).not_to receive(:new) + expect(::Projects::MarkForDeletionService).not_to receive(:perform_in) + + worker.perform + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hget('inactive_projects_deletion_warning_email_notified', + "project:#{inactive_large_project.id}")).to be_nil + end + end + + it 'invokes Projects::InactiveProjectsDeletionNotificationWorker for inactive projects and logs audit event' do + Gitlab::Redis::SharedState.with do |redis| + expect(redis).to receive(:hset).with('inactive_projects_deletion_warning_email_notified', + "project:#{inactive_large_project.id}", Date.current) + end + expect(::Projects::InactiveProjectsDeletionNotificationWorker).to receive(:perform_in).with( + delay, inactive_large_project.id, deletion_date).and_call_original + expect(::Projects::DestroyService).not_to receive(:new) + + expect { worker.perform } + .to change { AuditEvent.count }.by(1) + + expect(AuditEvent.last.details[:custom_message]) + .to eq("Project is scheduled to be deleted on #{deletion_date} due to inactivity.") + end + + context 'when adjourned_deletion_for_projects_and_groups feature is not available' do + before do + stub_licensed_features(adjourned_deletion_for_projects_and_groups: false) + end + + it 'invokes Projects::DestroyService for projects that are inactive even after being notified' do + Gitlab::Redis::SharedState.with do |redis| + redis.hset('inactive_projects_deletion_warning_email_notified', "project:#{inactive_large_project.id}", + 15.months.ago.to_date.to_s) + end + + expect(::Projects::InactiveProjectsDeletionNotificationWorker).not_to receive(:perform_in) + expect(::Projects::MarkForDeletionService).not_to receive(:perform_in) + expect(::Projects::DestroyService).to receive(:new).with(inactive_large_project, admin_user, {}) + .at_least(:once).and_call_original + + worker.perform + + expect(inactive_large_project.reload.pending_delete).to eq(true) + expect(inactive_large_project.reload.marked_for_deletion_at).to be_nil + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hget('inactive_projects_deletion_warning_email_notified', + "project:#{inactive_large_project.id}")).to be_nil + end + end + end + + context 'when adjourned_deletion_for_projects_and_groups feature is available' do + before do + stub_licensed_features(adjourned_deletion_for_projects_and_groups: true) + end + + context 'when adjourned_deletion_configured is not configured for the project' do + before do + group.namespace_settings.update!(delayed_project_removal: false) + end + + it 'invokes Projects::DestroyService if adjourned_deletion_configured not configured for the project' do + Gitlab::Redis::SharedState.with do |redis| + redis.hset('inactive_projects_deletion_warning_email_notified', "project:#{inactive_large_project.id}", + 15.months.ago.to_date.to_s) + end + + expect(::Projects::InactiveProjectsDeletionNotificationWorker).not_to receive(:perform_in) + expect(::Projects::MarkForDeletionService).not_to receive(:perform_in) + expect(::Projects::DestroyService).to receive(:new).with(inactive_large_project, admin_user, {}) + .at_least(:once).and_call_original + + worker.perform + + expect(inactive_large_project.reload.pending_delete).to eq(true) + expect(inactive_large_project.reload.marked_for_deletion_at).to be_nil + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hget('inactive_projects_deletion_warning_email_notified', + "project:#{inactive_large_project.id}")).to be_nil + end + end + end + + context 'when adjourned_deletion_configured is configured for the project' do + before do + group.namespace_settings.update!(delayed_project_removal: true) + end + + it 'invokes Projects::MarkForDeletionService for projects that are inactive even after being notified' do + Gitlab::Redis::SharedState.with do |redis| + redis.hset('inactive_projects_deletion_warning_email_notified', "project:#{inactive_large_project.id}", + 15.months.ago.to_date.to_s) + end + + expect(::Projects::InactiveProjectsDeletionNotificationWorker).not_to receive(:perform_in) + expect(::Projects::MarkForDeletionService).to receive(:new).with(inactive_large_project, admin_user, {}) + .and_call_original + + worker.perform + + expect(inactive_large_project.reload.pending_delete).to eq(false) + expect(inactive_large_project.reload.marked_for_deletion_at).not_to be_nil + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hget('inactive_projects_deletion_warning_email_notified', + "project:#{inactive_large_project.id}")).to be_nil + end + end + end + end + end +end diff --git a/lib/gitlab/inactive_projects_deletion_warning_tracker.rb b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb new file mode 100644 index 0000000000000000000000000000000000000000..f3f8e774b4b662fd2fc36871fe20311660f26d4d --- /dev/null +++ b/lib/gitlab/inactive_projects_deletion_warning_tracker.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +module Gitlab + class InactiveProjectsDeletionWarningTracker + attr_reader :project_id + + DELETION_TRACKING_REDIS_KEY = 'inactive_projects_deletion_warning_email_notified' + + # Redis key 'inactive_projects_deletion_warning_email_notified' is a hash. It stores the date when the + # deletion warning notification email was sent for an inactive project. The fields and values look like: + # {"project:1"=>"2022-04-22", "project:5"=>"2022-04-22", "project:7"=>"2022-04-25"} + # @return [Hash] + def self.notified_projects + Gitlab::Redis::SharedState.with do |redis| + redis.hgetall(DELETION_TRACKING_REDIS_KEY) + end + end + + def self.reset_all + Gitlab::Redis::SharedState.with do |redis| + redis.del(DELETION_TRACKING_REDIS_KEY) + end + end + + def initialize(project_id) + @project_id = project_id + end + + def notified? + Gitlab::Redis::SharedState.with do |redis| + redis.hexists(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}") + end + end + + def mark_notified + Gitlab::Redis::SharedState.with do |redis| + redis.hset(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}", Date.current) + end + end + + def reset + Gitlab::Redis::SharedState.with do |redis| + redis.hdel(DELETION_TRACKING_REDIS_KEY, "project:#{project_id}") + end + end + end +end diff --git a/spec/lib/gitlab/inactive_projects_deletion_warning_tracker_spec.rb b/spec/lib/gitlab/inactive_projects_deletion_warning_tracker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..4eb2388f3f7e5b410ec433833f99c9ec175a9e29 --- /dev/null +++ b/spec/lib/gitlab/inactive_projects_deletion_warning_tracker_spec.rb @@ -0,0 +1,67 @@ +# frozen_string_literal: true + +require "spec_helper" + +RSpec.describe Gitlab::InactiveProjectsDeletionWarningTracker do + let_it_be(:project_id) { 1 } + + describe '.notified_projects', :clean_gitlab_redis_shared_state do + before do + freeze_time do + Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).mark_notified + end + end + + it 'returns the list of projects for which deletion warning email has been sent' do + expected_hash = { "project:1" => "#{Date.current}" } + + expect(Gitlab::InactiveProjectsDeletionWarningTracker.notified_projects).to eq(expected_hash) + end + end + + describe '.reset_all' do + before do + Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).mark_notified + end + + it 'deletes all the projects for which deletion warning email was sent' do + Gitlab::InactiveProjectsDeletionWarningTracker.reset_all + + expect(Gitlab::InactiveProjectsDeletionWarningTracker.notified_projects).to eq({}) + end + end + + describe '#notified?' do + before do + Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).mark_notified + end + + it 'returns true if the project has already been notified' do + expect(Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).notified?).to eq(true) + end + + it 'returns false if the project has not been notified' do + expect(Gitlab::InactiveProjectsDeletionWarningTracker.new(2).notified?).to eq(false) + end + end + + describe '#mark_notified' do + it 'marks the project as being notified' do + Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).mark_notified + + expect(Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).notified?).to eq(true) + end + end + + describe '#reset' do + before do + Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).mark_notified + end + + it 'resets the project as not being notified' do + Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).reset + + expect(Gitlab::InactiveProjectsDeletionWarningTracker.new(project_id).notified?).to eq(false) + end + end +end diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 9f0f056d14e283575f0178be4b5ba4f4ab229d0a..3e8d22056532f7ecf22dd073048a85a0daf5c793 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -1344,5 +1344,17 @@ def expect_invalid it { is_expected.to validate_numericality_of(:inactive_projects_delete_after_months).is_greater_than(0) } it { is_expected.to validate_numericality_of(:inactive_projects_min_size_mb).is_greater_than_or_equal_to(0) } + + it "deletes the redis key used for tracking inactive projects deletion warning emails when setting is updated", + :clean_gitlab_redis_shared_state do + Gitlab::Redis::SharedState.with do |redis| + redis.hset("inactive_projects_deletion_warning_email_notified", "project:1", "2020-01-01") + end + + Gitlab::Redis::SharedState.with do |redis| + expect { setting.update!(inactive_projects_delete_after_months: 6) } + .to change { redis.hgetall('inactive_projects_deletion_warning_email_notified') }.to({}) + end + end end end diff --git a/spec/models/event_spec.rb b/spec/models/event_spec.rb index f099015e63e82a0407133feb230991c5d6e64dc0..8f61369958e28eb293ea098c4f4bcdcd60bc1c8b 100644 --- a/spec/models/event_spec.rb +++ b/spec/models/event_spec.rb @@ -834,7 +834,13 @@ def visible_to_all_except(*roles) end end - context 'when a project was updated more than 1 hour ago' do + context 'when a project was updated more than 1 hour ago', :clean_gitlab_redis_shared_state do + before do + ::Gitlab::Redis::SharedState.with do |redis| + redis.hset('inactive_projects_deletion_warning_email_notified', "project:#{project.id}", Date.current) + end + end + it 'updates the project' do project.touch(:last_activity_at, time: 1.year.ago) # rubocop: disable Rails/SkipsModelValidations @@ -845,6 +851,17 @@ def visible_to_all_except(*roles) expect(project.last_activity_at).to be_like_time(event.created_at) expect(project.updated_at).to be_like_time(event.created_at) end + + it "deletes the redis key for if the project was inactive" do + Gitlab::Redis::SharedState.with do |redis| + expect(redis).to receive(:hdel).with('inactive_projects_deletion_warning_email_notified', + "project:#{project.id}") + end + + project.touch(:last_activity_at, time: 1.year.ago) # rubocop: disable Rails/SkipsModelValidations + + create_push_event(project, project.first_owner) + end end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 7b01c653981bd5e5743e27be2b2e66d02c03b577..ed5b3d4e0bed31f5f3034957fe74360ad915a9c9 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -8,6 +8,7 @@ include ExternalAuthorizationServiceHelpers include ReloadHelpers include StubGitlabCalls + include ProjectHelpers using RSpec::Parameterized::TableSyntax let_it_be(:namespace) { create_default(:namespace).freeze } @@ -8267,6 +8268,28 @@ def has_external_wiki it_behaves_like 'returns true if project is inactive' end + describe '.inactive' do + before do + stub_application_setting(inactive_projects_min_size_mb: 5) + stub_application_setting(inactive_projects_send_warning_email_after_months: 12) + end + + it 'returns projects that are inactive' do + create_project_with_statistics.tap do |project| + project.update!(last_activity_at: Time.current) + end + create_project_with_statistics.tap do |project| + project.update!(last_activity_at: 13.months.ago) + end + inactive_large_project = create_project_with_statistics(with_data: true, size_multiplier: 2.gigabytes) + .tap { |project| project.update!(last_activity_at: 2.years.ago) } + create_project_with_statistics(with_data: true, size_multiplier: 2.gigabytes) + .tap { |project| project.update!(last_activity_at: 1.month.ago) } + + expect(described_class.inactive).to contain_exactly(inactive_large_project) + end + end + private def finish_job(export_job) diff --git a/spec/support/helpers/project_helpers.rb b/spec/support/helpers/project_helpers.rb index 89f0163b4b65ac254e307c889459971ea78bb89b..2ea6405e48c38bb1fb4dd5a082608fcb33d871c8 100644 --- a/spec/support/helpers/project_helpers.rb +++ b/spec/support/helpers/project_helpers.rb @@ -24,4 +24,20 @@ def update_feature_access_level(project, access_level) project.update!(params) end + + def create_project_with_statistics(namespace = nil, with_data: false, size_multiplier: 1) + project = namespace.present? ? create(:project, namespace: namespace) : create(:project) + project.tap do |p| + create(:project_statistics, project: p, with_data: with_data, size_multiplier: size_multiplier) + end + end + + def grace_months_after_deletion_notification + (::Gitlab::CurrentSettings.inactive_projects_delete_after_months - + ::Gitlab::CurrentSettings.inactive_projects_send_warning_email_after_months).months + end + + def deletion_date + Date.parse(grace_months_after_deletion_notification.from_now.to_s).to_s + end end diff --git a/spec/workers/every_sidekiq_worker_spec.rb b/spec/workers/every_sidekiq_worker_spec.rb index adee70fbf87fdbbab1d2fb73c661c2efa976fb06..ca858bcba193e9faf9204a45c584d8f08966ebb2 100644 --- a/spec/workers/every_sidekiq_worker_spec.rb +++ b/spec/workers/every_sidekiq_worker_spec.rb @@ -389,6 +389,7 @@ 'ProjectTemplateExportWorker' => false, 'ProjectUpdateRepositoryStorageWorker' => 3, 'Projects::GitGarbageCollectWorker' => false, + 'Projects::InactiveProjectsDeletionNotificationWorker' => 3, 'Projects::PostCreationWorker' => 3, 'Projects::ScheduleBulkRepositoryShardMovesWorker' => 3, 'Projects::UpdateRepositoryStorageWorker' => 3, diff --git a/spec/workers/projects/inactive_projects_deletion_cron_worker_spec.rb b/spec/workers/projects/inactive_projects_deletion_cron_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..0e7b4ea504c500fbe1bcb3be3044478b5b36c020 --- /dev/null +++ b/spec/workers/projects/inactive_projects_deletion_cron_worker_spec.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::InactiveProjectsDeletionCronWorker do + include ProjectHelpers + + describe "#perform" do + subject(:worker) { described_class.new } + + let_it_be(:admin_user) { create(:user, :admin) } + let_it_be(:non_admin_user) { create(:user) } + let_it_be(:new_blank_project) do + create_project_with_statistics.tap do |project| + project.update!(last_activity_at: Time.current) + end + end + + let_it_be(:inactive_blank_project) do + create_project_with_statistics.tap do |project| + project.update!(last_activity_at: 13.months.ago) + end + end + + let_it_be(:inactive_large_project) do + create_project_with_statistics(with_data: true, size_multiplier: 2.gigabytes) + .tap { |project| project.update!(last_activity_at: 2.years.ago) } + end + + let_it_be(:active_large_project) do + create_project_with_statistics(with_data: true, size_multiplier: 2.gigabytes) + .tap { |project| project.update!(last_activity_at: 1.month.ago) } + end + + before do + stub_application_setting(inactive_projects_min_size_mb: 5) + stub_application_setting(inactive_projects_send_warning_email_after_months: 12) + stub_application_setting(inactive_projects_delete_after_months: 14) + end + + context 'when delete inactive projects feature is disabled' do + before do + stub_application_setting(delete_inactive_projects: false) + end + + it 'does not invoke Projects::InactiveProjectsDeletionNotificationWorker' do + expect(::Projects::InactiveProjectsDeletionNotificationWorker).not_to receive(:perform_in) + expect(::Projects::DestroyService).not_to receive(:new) + + worker.perform + end + + it 'does not delete the inactive projects' do + worker.perform + + expect(inactive_large_project.reload.pending_delete).to eq(false) + end + end + + context 'when delete inactive projects feature is enabled' do + before do + stub_application_setting(delete_inactive_projects: true) + end + + context 'when feature flag is disabled' do + before do + stub_feature_flags(inactive_projects_deletion: false) + end + + it 'does not invoke Projects::InactiveProjectsDeletionNotificationWorker' do + expect(::Projects::InactiveProjectsDeletionNotificationWorker).not_to receive(:perform_in) + expect(::Projects::DestroyService).not_to receive(:new) + + worker.perform + end + + it 'does not delete the inactive projects' do + worker.perform + + expect(inactive_large_project.reload.pending_delete).to eq(false) + end + end + + context 'when feature flag is enabled', :clean_gitlab_redis_shared_state, :sidekiq_inline do + let_it_be(:delay) { anything } + + before do + stub_feature_flags(inactive_projects_deletion: true) + end + + it 'invokes Projects::InactiveProjectsDeletionNotificationWorker for inactive projects' do + Gitlab::Redis::SharedState.with do |redis| + expect(redis).to receive(:hset).with('inactive_projects_deletion_warning_email_notified', + "project:#{inactive_large_project.id}", Date.current) + end + expect(::Projects::InactiveProjectsDeletionNotificationWorker).to receive(:perform_in).with( + delay, inactive_large_project.id, deletion_date).and_call_original + expect(::Projects::DestroyService).not_to receive(:new) + + worker.perform + end + + it 'does not invoke InactiveProjectsDeletionNotificationWorker for already notified inactive projects' do + Gitlab::Redis::SharedState.with do |redis| + redis.hset('inactive_projects_deletion_warning_email_notified', "project:#{inactive_large_project.id}", + Date.current.to_s) + end + + expect(::Projects::InactiveProjectsDeletionNotificationWorker).not_to receive(:perform_in) + expect(::Projects::DestroyService).not_to receive(:new) + + worker.perform + end + + it 'invokes Projects::DestroyService for projects that are inactive even after being notified' do + Gitlab::Redis::SharedState.with do |redis| + redis.hset('inactive_projects_deletion_warning_email_notified', "project:#{inactive_large_project.id}", + 15.months.ago.to_date.to_s) + end + + expect(::Projects::InactiveProjectsDeletionNotificationWorker).not_to receive(:perform_in) + expect(::Projects::DestroyService).to receive(:new).with(inactive_large_project, admin_user, {}) + .at_least(:once).and_call_original + + worker.perform + + expect(inactive_large_project.reload.pending_delete).to eq(true) + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hget('inactive_projects_deletion_warning_email_notified', + "project:#{inactive_large_project.id}")).to be_nil + end + end + end + + it_behaves_like 'an idempotent worker' + end + end +end diff --git a/spec/workers/projects/inactive_projects_deletion_notification_worker_spec.rb b/spec/workers/projects/inactive_projects_deletion_notification_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..3ddfec0d346de48c9da0e5cbaf7fd280442296ba --- /dev/null +++ b/spec/workers/projects/inactive_projects_deletion_notification_worker_spec.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::InactiveProjectsDeletionNotificationWorker do + describe "#perform" do + subject(:worker) { described_class.new } + + let_it_be(:deletion_date) { Date.current } + let_it_be(:non_existing_project_id) { non_existing_record_id } + let_it_be(:project) { create(:project) } + + it 'invokes NotificationService and calls inactive_project_deletion_warning' do + expect_next_instance_of(NotificationService) do |notification| + expect(notification).to receive(:inactive_project_deletion_warning).with(project, deletion_date) + end + + worker.perform(project.id, deletion_date) + end + + it 'adds the project_id to redis key that tracks the deletion warning emails' do + worker.perform(project.id, deletion_date) + + Gitlab::Redis::SharedState.with do |redis| + expect(redis.hget('inactive_projects_deletion_warning_email_notified', + "project:#{project.id}")).to eq(Date.current.to_s) + end + end + + it 'rescues and logs the exception if project does not exist' do + expect(Gitlab::ErrorTracking).to receive(:log_exception).with(instance_of(ActiveRecord::RecordNotFound), + { project_id: non_existing_project_id }) + + worker.perform(non_existing_project_id, deletion_date) + end + + it_behaves_like 'an idempotent worker' do + let(:job_args) { [project.id, deletion_date] } + end + end +end