From 569557f02d71733a7231ac88bfd46827f3d83759 Mon Sep 17 00:00:00 2001 From: Stan Hu Date: Wed, 9 Feb 2022 00:45:40 -0800 Subject: [PATCH] Allow GC to run if deduplication service runs into an error We noticed that the Gitaly `FetchIntoObjectPool` RPC can fail in Gitaly Cluster. As a result, garbage collection was blocked, preventing cleanup from happening. We now allow GC to continue if GRPC either times out or raise some internal error. Relates to https://gitlab.com/gitlab-org/gitaly/-/issues/4022 Changelog: fixed --- .../projects/git_garbage_collect_worker.rb | 10 +++++++++- .../projects/git_garbage_collect_worker_spec.rb | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/app/workers/projects/git_garbage_collect_worker.rb b/app/workers/projects/git_garbage_collect_worker.rb index cf236f8b6603f6..d16583975fc902 100644 --- a/app/workers/projects/git_garbage_collect_worker.rb +++ b/app/workers/projects/git_garbage_collect_worker.rb @@ -16,7 +16,15 @@ def find_resource(id) def before_gitaly_call(task, resource) return unless gc?(task) - ::Projects::GitDeduplicationService.new(resource).execute + # Don't block garbage collection if we can't fetch into an object pool + # due to some gRPC error because we don't want to accumulate cruft. + # See https://gitlab.com/gitlab-org/gitaly/-/issues/4022. + begin + ::Projects::GitDeduplicationService.new(resource).execute + rescue Gitlab::Git::CommandTimedOut, GRPC::Internal => e + Gitlab::ErrorTracking.track_exception(e) + end + cleanup_orphan_lfs_file_references(resource) end diff --git a/spec/workers/projects/git_garbage_collect_worker_spec.rb b/spec/workers/projects/git_garbage_collect_worker_spec.rb index 7b54d7df4b2340..ae5671074436e9 100644 --- a/spec/workers/projects/git_garbage_collect_worker_spec.rb +++ b/spec/workers/projects/git_garbage_collect_worker_spec.rb @@ -32,6 +32,21 @@ subject.perform(*params) end + + context 'when deduplication service runs into a GRPC internal error' do + before do + allow_next_instance_of(::Projects::GitDeduplicationService) do |instance| + expect(instance).to receive(:execute).and_raise(GRPC::Internal) + end + end + + it_behaves_like 'can collect git garbage' do + let(:resource) { project } + let(:statistics_service_klass) { Projects::UpdateStatisticsService } + let(:statistics_keys) { [:repository_size, :lfs_objects_size] } + let(:expected_default_lease) { "projects:#{resource.id}" } + end + end end context 'LFS object garbage collection' do -- GitLab