From 2d72531a21f49309b9d1187b3d3c98dec7abe94e Mon Sep 17 00:00:00 2001 From: Kev Kloss Date: Fri, 19 Sep 2025 13:58:44 +0200 Subject: [PATCH 1/3] Speed up 13_comments seeder by up to 80% This parallelizes the `13_comments` seeder, reducing the duration by around 80% on macOS (~275 to ~55). As identified previously, this fixture seeder is a major contributor to the duration of all seeding. While we have added a `--fast` option to reset a seeded GDK database in less than a minute, seeding from source is still the default operation and also runs in CI. --- db/fixtures/development/13_comments.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/fixtures/development/13_comments.rb b/db/fixtures/development/13_comments.rb index f19beaefcb7a8f..56a712ad28c307 100644 --- a/db/fixtures/development/13_comments.rb +++ b/db/fixtures/development/13_comments.rb @@ -1,7 +1,7 @@ require './spec/support/sidekiq_middleware' Gitlab::Seeder.quiet do - Issue.find_each do |issue| + Parallel.each(Issue.preload(:project).all, in_processes: Parallel.processor_count) do |issue| project = issue.project project.team.users.each do |user| @@ -18,7 +18,7 @@ end end - MergeRequest.find_each do |mr| + Parallel.each(MergeRequest.all, in_processes: Parallel.processor_count) do |mr| project = mr.project project.team.users.each do |user| -- GitLab From f65e2cc1f529ca19e6f83c433e020f319ff84c16 Mon Sep 17 00:00:00 2001 From: Kev Kloss Date: Fri, 26 Sep 2025 11:42:51 +0200 Subject: [PATCH 2/3] Add parallel_each helper --- db/fixtures/development/13_comments.rb | 4 ++-- lib/gitlab/seeder.rb | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/db/fixtures/development/13_comments.rb b/db/fixtures/development/13_comments.rb index 56a712ad28c307..519d91af0c11c3 100644 --- a/db/fixtures/development/13_comments.rb +++ b/db/fixtures/development/13_comments.rb @@ -1,7 +1,7 @@ require './spec/support/sidekiq_middleware' Gitlab::Seeder.quiet do - Parallel.each(Issue.preload(:project).all, in_processes: Parallel.processor_count) do |issue| + Gitlab::Seeder.parallel_each(Issue.preload(:project).all) do |issue| project = issue.project project.team.users.each do |user| @@ -18,7 +18,7 @@ end end - Parallel.each(MergeRequest.all, in_processes: Parallel.processor_count) do |mr| + Gitlab::Seeder.parallel_each(MergeRequest.all) do |mr| project = mr.project project.team.users.each do |user| diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index e2c8c0ccdd0112..54bf15a61ee52e 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -45,6 +45,15 @@ module UserSeed end end + def self.parallel_each(array) + raise 'Nested Gitlab::Seeder.parallel_each calls are not allowed.' if @parallel + + @parallel = true + Parallel.each(array, in_processes: Parallel.processor_count) { |record| yield(record) } + ensure + @parallel = false + end + def self.log_message(message) puts "#{Time.current}: #{message}" end -- GitLab From ed8b9857a04584d122385a91f1f2a8db2062752f Mon Sep 17 00:00:00 2001 From: Kev Kloss Date: Tue, 30 Sep 2025 11:23:39 +0200 Subject: [PATCH 3/3] Clear Gitaly stubs before forking --- lib/gitlab/seeder.rb | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/gitlab/seeder.rb b/lib/gitlab/seeder.rb index 54bf15a61ee52e..8b1cc678522077 100644 --- a/lib/gitlab/seeder.rb +++ b/lib/gitlab/seeder.rb @@ -49,6 +49,12 @@ def self.parallel_each(array) raise 'Nested Gitlab::Seeder.parallel_each calls are not allowed.' if @parallel @parallel = true + + # Closing gRPC channels before forking prevents segfaults when Ruby + # finalizes inherited gRPC objects in the child process. + # See https://gitlab.com/gitlab-org/gitlab/-/issues/480716#note_2787476778 + Gitlab::GitalyClient.clear_stubs! + Parallel.each(array, in_processes: Parallel.processor_count) { |record| yield(record) } ensure @parallel = false -- GitLab