diff --git a/app/models/ci/namespace_mirror.rb b/app/models/ci/namespace_mirror.rb new file mode 100644 index 0000000000000000000000000000000000000000..833d25055a1c9daef3355ed8c55e0de0fc5b8066 --- /dev/null +++ b/app/models/ci/namespace_mirror.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Ci + # This model represents a record in a shadow table of the main database's namespaces table. + # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN. + class NamespaceMirror < ApplicationRecord + scope :contains_namespaces, -> (ids) { + where('traversal_ids @> ARRAY[?]::int[]', ids.join(',')) + } + + class << self + def sync!(event) + upsert({ namespace_id: event.namespace_id, traversal_ids: event.traversal_ids }, + unique_by: :namespace_id) + + sync_other_namespaces!(event) + end + + private + + def sync_other_namespaces!(event) + contains_namespaces([event.namespace_id]) + .where.not(namespace_id: event.namespace_id) + .update_all( + "traversal_ids = ARRAY[#{event.traversal_ids.join(',')}]::int[] || traversal_ids[array_position(traversal_ids, #{event.namespace_id}) + 1:]" + ) + end + end + end +end diff --git a/app/models/ci/project_mirror.rb b/app/models/ci/project_mirror.rb new file mode 100644 index 0000000000000000000000000000000000000000..f6a15cae37fc988c8c5c9c5c7dcf5e95425e812a --- /dev/null +++ b/app/models/ci/project_mirror.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Ci + # This model represents a shadow table of the main database's projects table. + # It allows us to navigate the project and namespace hierarchy on the ci database. + class ProjectMirror < ApplicationRecord + class << self + def sync!(event) + upsert({ project_id: event.project_id, namespace_id: event.namespace_id }, + unique_by: :project_id) + end + end + end +end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index c4665ca68283e4830d82a6a38399c09ed2347c94..dbd9aa9ba6d5cce44a93dd0026dc5a2eab0346b4 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -104,6 +104,8 @@ class Namespace < ApplicationRecord delegate :name, to: :owner, allow_nil: true, prefix: true delegate :avatar_url, to: :owner, allow_nil: true + after_save :schedule_ci_hierarchy_update, if: -> { saved_change_to_id? || saved_change_to_parent_id? } + after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } before_create :sync_share_with_group_lock_with_parent @@ -613,6 +615,17 @@ def write_projects_repository_config def enforce_minimum_path_length? path_changed? && !project_namespace? end + + def schedule_ci_hierarchy_update + Namespaces::SyncEvent.insert({ + namespace_id: id, + traversal_ids: self_and_ancestor_ids(hierarchy_order: :desc) + }) + + run_after_commit do + Ci::ProcessNamespaceSyncEventsWorker.perform_async + end + end end Namespace.prepend_mod_with('Namespace') diff --git a/app/models/namespaces/sync_event.rb b/app/models/namespaces/sync_event.rb new file mode 100644 index 0000000000000000000000000000000000000000..2b8f7d35a9e187290fd418e14c5441352ec909d3 --- /dev/null +++ b/app/models/namespaces/sync_event.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# This model serves to keep track of changes to the namespaces table in the main database, and allowing to safely +# replicate these changes to other databases. +class Namespaces::SyncEvent < ApplicationRecord + include EachBatch + + self.table_name = 'namespaces_sync_events' +end diff --git a/app/models/project.rb b/app/models/project.rb index a4e84b412ec27fb2bab52a42658368fc7a62f51c..90f8e865d2153f796cd3fb063762e5cc73c8d6ae 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -101,6 +101,8 @@ class Project < ApplicationRecord after_save :update_project_statistics, if: :saved_change_to_namespace_id? + after_save :schedule_ci_hierarchy_update, if: -> { saved_change_to_id? || saved_change_to_namespace_id? } + after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? } after_save :save_topics @@ -2959,6 +2961,17 @@ def sync_attributes(project_namespace) project_namespace.shared_runners_enabled = shared_runners_enabled project_namespace.visibility_level = visibility_level end + + def schedule_ci_hierarchy_update + Projects::SyncEvent.insert({ + project_id: id, + namespace_id: namespace_id + }) + + run_after_commit do + Ci::ProcessProjectSyncEventsWorker.perform_async + end + end end Project.prepend_mod_with('Project') diff --git a/app/models/projects/sync_event.rb b/app/models/projects/sync_event.rb new file mode 100644 index 0000000000000000000000000000000000000000..af90ff29009412837ccd8af4d03759e7d60d5664 --- /dev/null +++ b/app/models/projects/sync_event.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +# This model serves to keep track of changes to the namespaces table in the main database as they relate to projects, +# allowing to safely replicate changes to other databases. +class Projects::SyncEvent < ApplicationRecord + include EachBatch + + self.table_name = 'projects_sync_events' +end diff --git a/app/services/ci/process_sync_events_service.rb b/app/services/ci/process_sync_events_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..d56777cf07f97d447f262df36541d558deaa81c5 --- /dev/null +++ b/app/services/ci/process_sync_events_service.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Ci + class ProcessSyncEventsService + BATCH_SIZE = 100 + + def initialize(sync_event_class, sync_class) + @sync_event_class = sync_event_class + @sync_class = sync_class + end + + def execute + @sync_event_class.each_batch(of: BATCH_SIZE) do |events| + events.each { |event| @sync_class.sync!(event) } + events.delete_all + + # TODO: This is the old query, should we use something like this: + # select_query = @sync_event_class + # .id_in(batch) + # .lock!('FOR UPDATE SKIP LOCKED') + # .select(:id) + + # @sync_event_class.where(id: select_query).delete_all + end + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 733a878907c573cb453114410aa0793c79f8cd92..1c1a743146889643683712475695f05a1bc83024 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1960,6 +1960,24 @@ :weight: 1 :idempotent: true :tags: [] +- :name: ci_process_namespace_sync_events + :worker_name: Ci::ProcessNamespaceSyncEventsWorker + :feature_category: :subgroups + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: ci_process_project_sync_events + :worker_name: Ci::ProcessProjectSyncEventsWorker + :feature_category: :subgroups + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: create_commit_signature :worker_name: CreateCommitSignatureWorker :feature_category: :source_code_management diff --git a/app/workers/ci/process_namespace_sync_events_worker.rb b/app/workers/ci/process_namespace_sync_events_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..2411e4d7afbba47fe985b94807706800c5e4718b --- /dev/null +++ b/app/workers/ci/process_namespace_sync_events_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Ci + class ProcessNamespaceSyncEventsWorker + include ApplicationWorker + + data_consistency :always + + feature_category :subgroups + + deduplicate :until_executed + idempotent! + + def perform + Ci::ProcessSyncEventsService.new(::Namespaces::SyncEvent, ::Ci::NamespaceMirror).execute + end + end +end diff --git a/app/workers/ci/process_project_sync_events_worker.rb b/app/workers/ci/process_project_sync_events_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..b981dbcc303405f6ee18c96e9e6769c79769426c --- /dev/null +++ b/app/workers/ci/process_project_sync_events_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Ci + class ProcessProjectSyncEventsWorker + include ApplicationWorker + + data_consistency :always + + feature_category :subgroups + + deduplicate :until_executed + idempotent! + + def perform + Ci::ProcessSyncEventsService.new(::Projects::SyncEvent, ::Ci::ProjectMirror).execute + end + end +end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 7e6bcc35e60170a4480ab520a6b4ae50176942a4..a0024ebaa11efc115db83546df451a2c05ebee13 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -73,6 +73,10 @@ - 1 - - ci_delete_objects - 1 +- - ci_process_namespace_sync_events + - 1 +- - ci_process_project_sync_events + - 1 - - ci_upstream_projects_subscriptions_cleanup - 1 - - container_repository diff --git a/db/migrate/20211011140930_create_ci_namespace_mirrors.rb b/db/migrate/20211011140930_create_ci_namespace_mirrors.rb new file mode 100644 index 0000000000000000000000000000000000000000..cf324e128919e6efa20462ed968c31a26ffaaad5 --- /dev/null +++ b/db/migrate/20211011140930_create_ci_namespace_mirrors.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CreateCiNamespaceMirrors < Gitlab::Database::Migration[1.0] + TABLE_NAME = :ci_namespace_mirrors + INDEX_NAME = "index_gin_#{TABLE_NAME}_on_traversal_ids" + + def change + create_table TABLE_NAME, id: false do |t| + t.integer :namespace_id, primary_key: true, null: false, index: false + t.integer :traversal_ids, array: true, default: [], null: false + + t.index :traversal_ids, name: INDEX_NAME, using: :gin + end + end +end diff --git a/db/migrate/20211011140931_create_ci_project_mirrors.rb b/db/migrate/20211011140931_create_ci_project_mirrors.rb new file mode 100644 index 0000000000000000000000000000000000000000..d55d5933fb6bab193201aab5ed2ec8eafbedb134 --- /dev/null +++ b/db/migrate/20211011140931_create_ci_project_mirrors.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class CreateCiProjectMirrors < Gitlab::Database::Migration[1.0] + TABLE_NAME = :ci_project_mirrors + + def change + create_table TABLE_NAME, id: false do |t| + t.integer :project_id, primary_key: true, null: false, index: false + t.integer :namespace_id, null: false, index: true + end + end +end diff --git a/db/migrate/20211011140932_create_namespaces_sync_events.rb b/db/migrate/20211011140932_create_namespaces_sync_events.rb new file mode 100644 index 0000000000000000000000000000000000000000..0b4d7ba82b4e8bf416bf65c87bf23e4aeb43f2df --- /dev/null +++ b/db/migrate/20211011140932_create_namespaces_sync_events.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class CreateNamespacesSyncEvents < Gitlab::Database::Migration[1.0] + def change + create_table :namespaces_sync_events do |t| + t.references :namespace, null: false, index: true, foreign_key: { on_delete: :cascade } + t.integer :traversal_ids, array: true, default: [], null: false + end + end +end diff --git a/db/migrate/20211011141239_create_projects_sync_events.rb b/db/migrate/20211011141239_create_projects_sync_events.rb new file mode 100644 index 0000000000000000000000000000000000000000..491e405f6985a3bc695b0245bf0dd78a1270c806 --- /dev/null +++ b/db/migrate/20211011141239_create_projects_sync_events.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class CreateProjectsSyncEvents < Gitlab::Database::Migration[1.0] + def change + create_table :projects_sync_events do |t| + t.references :project, null: false, index: true, foreign_key: { on_delete: :cascade } + t.integer :namespace_id, null: false + end + end +end diff --git a/db/post_migrate/20211011141240_schedule_backfill_ci_namespace_mirrors.rb b/db/post_migrate/20211011141240_schedule_backfill_ci_namespace_mirrors.rb new file mode 100644 index 0000000000000000000000000000000000000000..1709c09eaf7afe91bdb3cb77093553f3026944ef --- /dev/null +++ b/db/post_migrate/20211011141240_schedule_backfill_ci_namespace_mirrors.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class ScheduleBackfillCiNamespaceMirrors < Gitlab::Database::Migration[1.0] + MIGRATION = 'BackfillCiNamespaceMirrors' + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 100 + DELAY_INTERVAL = 2.minutes + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::BackfillCiNamespaceMirrors::Namespace.base_query, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + other_job_arguments: [SUB_BATCH_SIZE], + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20211011141241_schedule_backfill_ci_project_mirrors.rb b/db/post_migrate/20211011141241_schedule_backfill_ci_project_mirrors.rb new file mode 100644 index 0000000000000000000000000000000000000000..802ff81a42f2b4526befdb1d4fec0841355e8c6b --- /dev/null +++ b/db/post_migrate/20211011141241_schedule_backfill_ci_project_mirrors.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +class ScheduleBackfillCiProjectMirrors < Gitlab::Database::Migration[1.0] + MIGRATION = 'BackfillCiProjectMirrors' + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 100 + DELAY_INTERVAL = 2.minutes + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::BackfillCiProjectMirrors::Project.base_query, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + other_job_arguments: [SUB_BATCH_SIZE], + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20211011140930 b/db/schema_migrations/20211011140930 new file mode 100644 index 0000000000000000000000000000000000000000..6347ee5d51d84d730d8203eb87c586c2e46f1d24 --- /dev/null +++ b/db/schema_migrations/20211011140930 @@ -0,0 +1 @@ +cdae819e8de3b5ad721014376bfd9af97a45e953e2d345daf62784f986a5eb31 \ No newline at end of file diff --git a/db/schema_migrations/20211011140931 b/db/schema_migrations/20211011140931 new file mode 100644 index 0000000000000000000000000000000000000000..c959d97074e02d7daf41a54dfe12cbe8d3d2bdf3 --- /dev/null +++ b/db/schema_migrations/20211011140931 @@ -0,0 +1 @@ +7e51eb4443fd74da9bef4d9c1c3cc40376c311abbc05ca7871f725fada79b48a \ No newline at end of file diff --git a/db/schema_migrations/20211011140932 b/db/schema_migrations/20211011140932 new file mode 100644 index 0000000000000000000000000000000000000000..af0e000b9f3e2a5512c10c788ec787df1596e86c --- /dev/null +++ b/db/schema_migrations/20211011140932 @@ -0,0 +1 @@ +0209db1e7be48bcbf0e52b451d37da0ef2ecadd567cdfa47907fc5032c258a27 \ No newline at end of file diff --git a/db/schema_migrations/20211011141239 b/db/schema_migrations/20211011141239 new file mode 100644 index 0000000000000000000000000000000000000000..f215f234a7e1a5528b28235bb5dec434dc7ef763 --- /dev/null +++ b/db/schema_migrations/20211011141239 @@ -0,0 +1 @@ +bc0ae055b331801fbe020c12a66e4e6ae790780121bfd66fd161093c94c7a84a \ No newline at end of file diff --git a/db/schema_migrations/20211011141240 b/db/schema_migrations/20211011141240 new file mode 100644 index 0000000000000000000000000000000000000000..ee736c5e0267cf75a5cf5826ad54f1f0448c3106 --- /dev/null +++ b/db/schema_migrations/20211011141240 @@ -0,0 +1 @@ +f5dad108d30fe1d3a36027cd3f6fd7db006444892e5ec145921117d490be11c7 \ No newline at end of file diff --git a/db/schema_migrations/20211011141241 b/db/schema_migrations/20211011141241 new file mode 100644 index 0000000000000000000000000000000000000000..33c6386425da192feb59f9255f94e4ab9e1051ac --- /dev/null +++ b/db/schema_migrations/20211011141241 @@ -0,0 +1 @@ +347aa02789f2016271cd669de005bb722283a4c59e48a2ca831342e123e38b27 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 3db0f0c14e92328c96681176e62515f261f65e12..7b967e6e583e06f9b3b7c701c3594e0ec5f97fec 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11763,6 +11763,21 @@ CREATE SEQUENCE ci_minutes_additional_packs_id_seq ALTER SEQUENCE ci_minutes_additional_packs_id_seq OWNED BY ci_minutes_additional_packs.id; +CREATE TABLE ci_namespace_mirrors ( + namespace_id integer NOT NULL, + traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL +); + +CREATE SEQUENCE ci_namespace_mirrors_namespace_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ci_namespace_mirrors_namespace_id_seq OWNED BY ci_namespace_mirrors.namespace_id; + CREATE TABLE ci_namespace_monthly_usages ( id bigint NOT NULL, namespace_id bigint NOT NULL, @@ -12011,6 +12026,21 @@ CREATE SEQUENCE ci_platform_metrics_id_seq ALTER SEQUENCE ci_platform_metrics_id_seq OWNED BY ci_platform_metrics.id; +CREATE TABLE ci_project_mirrors ( + project_id integer NOT NULL, + namespace_id integer NOT NULL +); + +CREATE SEQUENCE ci_project_mirrors_project_id_seq + AS integer + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ci_project_mirrors_project_id_seq OWNED BY ci_project_mirrors.project_id; + CREATE TABLE ci_project_monthly_usages ( id bigint NOT NULL, project_id bigint NOT NULL, @@ -16407,6 +16437,21 @@ CREATE SEQUENCE namespaces_id_seq ALTER SEQUENCE namespaces_id_seq OWNED BY namespaces.id; +CREATE TABLE namespaces_sync_events ( + id bigint NOT NULL, + namespace_id bigint NOT NULL, + traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL +); + +CREATE SEQUENCE namespaces_sync_events_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE namespaces_sync_events_id_seq OWNED BY namespaces_sync_events.id; + CREATE TABLE note_diff_files ( id integer NOT NULL, diff_note_id integer NOT NULL, @@ -18459,6 +18504,21 @@ CREATE SEQUENCE projects_id_seq ALTER SEQUENCE projects_id_seq OWNED BY projects.id; +CREATE TABLE projects_sync_events ( + id bigint NOT NULL, + project_id bigint NOT NULL, + namespace_id integer NOT NULL +); + +CREATE SEQUENCE projects_sync_events_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE projects_sync_events_id_seq OWNED BY projects_sync_events.id; + CREATE TABLE prometheus_alert_events ( id bigint NOT NULL, project_id integer NOT NULL, @@ -21259,6 +21319,8 @@ ALTER TABLE ONLY ci_job_variables ALTER COLUMN id SET DEFAULT nextval('ci_job_va ALTER TABLE ONLY ci_minutes_additional_packs ALTER COLUMN id SET DEFAULT nextval('ci_minutes_additional_packs_id_seq'::regclass); +ALTER TABLE ONLY ci_namespace_mirrors ALTER COLUMN namespace_id SET DEFAULT nextval('ci_namespace_mirrors_namespace_id_seq'::regclass); + ALTER TABLE ONLY ci_namespace_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_namespace_monthly_usages_id_seq'::regclass); ALTER TABLE ONLY ci_pending_builds ALTER COLUMN id SET DEFAULT nextval('ci_pending_builds_id_seq'::regclass); @@ -21281,6 +21343,8 @@ ALTER TABLE ONLY ci_pipelines_config ALTER COLUMN pipeline_id SET DEFAULT nextva ALTER TABLE ONLY ci_platform_metrics ALTER COLUMN id SET DEFAULT nextval('ci_platform_metrics_id_seq'::regclass); +ALTER TABLE ONLY ci_project_mirrors ALTER COLUMN project_id SET DEFAULT nextval('ci_project_mirrors_project_id_seq'::regclass); + ALTER TABLE ONLY ci_project_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_project_monthly_usages_id_seq'::regclass); ALTER TABLE ONLY ci_refs ALTER COLUMN id SET DEFAULT nextval('ci_refs_id_seq'::regclass); @@ -21669,6 +21733,8 @@ ALTER TABLE ONLY namespace_statistics ALTER COLUMN id SET DEFAULT nextval('names ALTER TABLE ONLY namespaces ALTER COLUMN id SET DEFAULT nextval('namespaces_id_seq'::regclass); +ALTER TABLE ONLY namespaces_sync_events ALTER COLUMN id SET DEFAULT nextval('namespaces_sync_events_id_seq'::regclass); + ALTER TABLE ONLY note_diff_files ALTER COLUMN id SET DEFAULT nextval('note_diff_files_id_seq'::regclass); ALTER TABLE ONLY notes ALTER COLUMN id SET DEFAULT nextval('notes_id_seq'::regclass); @@ -21819,6 +21885,8 @@ ALTER TABLE ONLY project_tracing_settings ALTER COLUMN id SET DEFAULT nextval('p ALTER TABLE ONLY projects ALTER COLUMN id SET DEFAULT nextval('projects_id_seq'::regclass); +ALTER TABLE ONLY projects_sync_events ALTER COLUMN id SET DEFAULT nextval('projects_sync_events_id_seq'::regclass); + ALTER TABLE ONLY prometheus_alert_events ALTER COLUMN id SET DEFAULT nextval('prometheus_alert_events_id_seq'::regclass); ALTER TABLE ONLY prometheus_alerts ALTER COLUMN id SET DEFAULT nextval('prometheus_alerts_id_seq'::regclass); @@ -22722,6 +22790,9 @@ ALTER TABLE ONLY ci_job_variables ALTER TABLE ONLY ci_minutes_additional_packs ADD CONSTRAINT ci_minutes_additional_packs_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_namespace_mirrors + ADD CONSTRAINT ci_namespace_mirrors_pkey PRIMARY KEY (namespace_id); + ALTER TABLE ONLY ci_namespace_monthly_usages ADD CONSTRAINT ci_namespace_monthly_usages_pkey PRIMARY KEY (id); @@ -22755,6 +22826,9 @@ ALTER TABLE ONLY ci_pipelines ALTER TABLE ONLY ci_platform_metrics ADD CONSTRAINT ci_platform_metrics_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_project_mirrors + ADD CONSTRAINT ci_project_mirrors_pkey PRIMARY KEY (project_id); + ALTER TABLE ONLY ci_project_monthly_usages ADD CONSTRAINT ci_project_monthly_usages_pkey PRIMARY KEY (id); @@ -23424,6 +23498,9 @@ ALTER TABLE ONLY namespace_statistics ALTER TABLE ONLY namespaces ADD CONSTRAINT namespaces_pkey PRIMARY KEY (id); +ALTER TABLE ONLY namespaces_sync_events + ADD CONSTRAINT namespaces_sync_events_pkey PRIMARY KEY (id); + ALTER TABLE ONLY note_diff_files ADD CONSTRAINT note_diff_files_pkey PRIMARY KEY (id); @@ -23694,6 +23771,9 @@ ALTER TABLE ONLY project_tracing_settings ALTER TABLE ONLY projects ADD CONSTRAINT projects_pkey PRIMARY KEY (id); +ALTER TABLE ONLY projects_sync_events + ADD CONSTRAINT projects_sync_events_pkey PRIMARY KEY (id); + ALTER TABLE ONLY prometheus_alert_events ADD CONSTRAINT prometheus_alert_events_pkey PRIMARY KEY (id); @@ -25420,6 +25500,8 @@ CREATE INDEX index_ci_pipelines_on_user_id_and_created_at_and_source ON ci_pipel CREATE INDEX index_ci_pipelines_on_user_id_and_id_and_cancelable_status ON ci_pipelines USING btree (user_id, id) WHERE ((status)::text = ANY (ARRAY[('running'::character varying)::text, ('waiting_for_resource'::character varying)::text, ('preparing'::character varying)::text, ('pending'::character varying)::text, ('created'::character varying)::text, ('scheduled'::character varying)::text])); +CREATE INDEX index_ci_project_mirrors_on_namespace_id ON ci_project_mirrors USING btree (namespace_id); + CREATE UNIQUE INDEX index_ci_project_monthly_usages_on_project_id_and_date ON ci_project_monthly_usages USING btree (project_id, date); CREATE UNIQUE INDEX index_ci_refs_on_project_id_and_ref_path ON ci_refs USING btree (project_id, ref_path); @@ -25996,6 +26078,8 @@ CREATE INDEX index_geo_repository_updated_events_on_source ON geo_repository_upd CREATE INDEX index_geo_reset_checksum_events_on_project_id ON geo_reset_checksum_events USING btree (project_id); +CREATE INDEX index_gin_ci_namespace_mirrors_on_traversal_ids ON ci_namespace_mirrors USING gin (traversal_ids); + CREATE INDEX index_gin_ci_pending_builds_on_namespace_traversal_ids ON ci_pending_builds USING gin (namespace_traversal_ids); CREATE INDEX index_gitlab_subscription_histories_on_gitlab_subscription_id ON gitlab_subscription_histories USING btree (gitlab_subscription_id); @@ -26556,6 +26640,8 @@ CREATE INDEX index_namespaces_on_type_and_id ON namespaces USING btree (type, id CREATE INDEX index_namespaces_public_groups_name_id ON namespaces USING btree (name, id) WHERE (((type)::text = 'Group'::text) AND (visibility_level = 20)); +CREATE INDEX index_namespaces_sync_events_on_namespace_id ON namespaces_sync_events USING btree (namespace_id); + CREATE INDEX index_non_requested_project_members_on_source_id_and_type ON members USING btree (source_id, source_type) WHERE ((requested_at IS NULL) AND ((type)::text = 'ProjectMember'::text)); CREATE UNIQUE INDEX index_note_diff_files_on_diff_note_id ON note_diff_files USING btree (diff_note_id); @@ -27016,6 +27102,8 @@ CREATE INDEX index_projects_on_star_count ON projects USING btree (star_count); CREATE INDEX index_projects_on_updated_at_and_id ON projects USING btree (updated_at, id); +CREATE INDEX index_projects_sync_events_on_project_id ON projects_sync_events USING btree (project_id); + CREATE UNIQUE INDEX index_prometheus_alert_event_scoped_payload_key ON prometheus_alert_events USING btree (prometheus_alert_id, payload_key); CREATE INDEX index_prometheus_alert_events_on_project_id_and_status ON prometheus_alert_events USING btree (project_id, status); @@ -30621,6 +30709,9 @@ ALTER TABLE ONLY gpg_keys ALTER TABLE ONLY analytics_language_trend_repository_languages ADD CONSTRAINT fk_rails_9d851d566c FOREIGN KEY (programming_language_id) REFERENCES programming_languages(id) ON DELETE CASCADE; +ALTER TABLE ONLY namespaces_sync_events + ADD CONSTRAINT fk_rails_9da32a0431 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY badges ADD CONSTRAINT fk_rails_9df4a56538 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; @@ -30795,6 +30886,9 @@ ALTER TABLE ONLY security_findings ALTER TABLE ONLY packages_debian_project_component_files ADD CONSTRAINT fk_rails_bbe9ebfbd9 FOREIGN KEY (component_id) REFERENCES packages_debian_project_components(id) ON DELETE RESTRICT; +ALTER TABLE ONLY projects_sync_events + ADD CONSTRAINT fk_rails_bbf0eef59f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY approval_merge_request_rules_users ADD CONSTRAINT fk_rails_bc8972fa55 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; diff --git a/ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb b/ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb index 929f59a85ad2cf2b12fb009ebb3afce980f60c64..e0742e49a6be7a570e20412d4457c93ab7cd372b 100644 --- a/ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb +++ b/ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb @@ -15,7 +15,7 @@ described_class.uncached_data end - expect(recorder.count).to eq(61) + expect(recorder.count).to eq(66) end end end diff --git a/ee/spec/models/ci/daily_build_group_report_result_spec.rb b/ee/spec/models/ci/daily_build_group_report_result_spec.rb index d622e92d963907ec78e91f88274aa6be69d8c130..e647c58516280b6ea8c1fc955c3367ca0a952299 100644 --- a/ee/spec/models/ci/daily_build_group_report_result_spec.rb +++ b/ee/spec/models/ci/daily_build_group_report_result_spec.rb @@ -100,8 +100,9 @@ end context 'when group has projects with several coverage' do - let!(:project_2) { create(:project) } - let!(:group) { create(:group, projects: [project, project_2]) } + let_it_be(:project_2) { create(:project) } + let_it_be(:group) { create(:group, projects: [project, project_2]) } + let!(:coverage_1) { create(:ci_daily_build_group_report_result, project: project) } let!(:coverage_2) { create(:ci_daily_build_group_report_result, project: project_2, group_name: 'karma') } diff --git a/lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb b/lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb new file mode 100644 index 0000000000000000000000000000000000000000..4687aefe208572415ead3618c3b90fbc3e4fa6fd --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_namespace_mirrors.rb @@ -0,0 +1,78 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to create ci_namespace_mirrors entries in sub-batches, relative to all namespaces + # that don't yet have a match in ci_namespace_mirrors. + class BackfillCiNamespaceMirrors + class Namespace < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'namespaces' + self.inheritance_column = nil + + scope :base_query, -> do + select(:id, :parent_id) + end + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + batch_query = Namespace.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + ranged_query = Namespace.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + INSERT INTO ci_namespace_mirrors (namespace_id, traversal_ids) + #{insert_values(ranged_query)} + ON CONFLICT (namespace_id) DO NOTHING + SQL + ActiveRecord::Base.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + # We have to add all arguments when marking a job as succeeded as they + # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + def insert_values(ranged_query) + calculated_traversal_ids(ranged_query.allow_cross_joins_across_databases(url: '')) + end + + # Calculate the ancestor path for a given set of namespaces. + def calculated_traversal_ids(batch) + <<~SQL + WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS ( + ( + SELECT batch.id, batch.id, batch.parent_id, 1 + FROM (#{batch.to_sql}) AS batch + ) + UNION ALL + ( + SELECT cte.source_id, n.id, n.parent_id, cte.height+1 + FROM namespaces n, cte + WHERE n.id = cte.parent_id + ) + ) + SELECT flat_hierarchy.source_id as namespace_id, + array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids + FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy + GROUP BY flat_hierarchy.source_id + SQL + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillCiNamespaces', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_ci_project_mirrors.rb b/lib/gitlab/background_migration/backfill_ci_project_mirrors.rb new file mode 100644 index 0000000000000000000000000000000000000000..00ca685a76f8757810786fc8978d00c84765464b --- /dev/null +++ b/lib/gitlab/background_migration/backfill_ci_project_mirrors.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to set ci_project_mirrors.namespace_id in sub-batches, of all projects + # that don't yet have a match in ci_project_mirrors. + class BackfillCiProjectMirrors + class Project < ActiveRecord::Base # rubocop:disable Style/Documentation + include ::EachBatch + + self.table_name = 'projects' + + scope :base_query, -> do + select(:id, :namespace_id) + end + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + batch_query = Project.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(projects.id), max(projects.id)')).first + ranged_query = Project.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + INSERT INTO ci_project_mirrors (project_id, namespace_id) + #{insert_values(ranged_query)} + ON CONFLICT (project_id) DO NOTHING + SQL + ActiveRecord::Base.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + # We have to add all arguments when marking a job as succeeded as they + # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + def insert_values(ranged_query) + ranged_query.allow_cross_joins_across_databases(url: '').to_sql + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillCiProjects', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 5695f2f1c14ec917f9a44ca36d8502fd2b3e4337..bf695693d31426797febb08f923b9c8e43c6660b 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -86,6 +86,7 @@ ci_job_token_project_scope_links: :gitlab_ci ci_job_variables: :gitlab_ci ci_minutes_additional_packs: :gitlab_ci ci_namespace_monthly_usages: :gitlab_ci +ci_namespace_mirrors: :gitlab_ci ci_pending_builds: :gitlab_ci ci_pipeline_artifacts: :gitlab_ci ci_pipeline_chat_data: :gitlab_ci @@ -97,6 +98,7 @@ ci_pipelines: :gitlab_ci ci_pipeline_variables: :gitlab_ci ci_platform_metrics: :gitlab_ci ci_project_monthly_usages: :gitlab_ci +ci_project_mirrors: :gitlab_ci ci_refs: :gitlab_ci ci_resource_groups: :gitlab_ci ci_resources: :gitlab_ci @@ -316,6 +318,7 @@ namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main namespace_settings: :gitlab_main namespaces: :gitlab_main +namespaces_sync_events: :gitlab_main namespace_statistics: :gitlab_main note_diff_files: :gitlab_main notes: :gitlab_main @@ -410,6 +413,7 @@ project_repository_storage_moves: :gitlab_main project_security_settings: :gitlab_main project_settings: :gitlab_main projects: :gitlab_main +projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main project_tracing_settings: :gitlab_main diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 9be09179c6cf40c7f6b548705cf6e8322b463c65..de20fbb3772fb67b993636148c52cdf37f0fc49b 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -27,9 +27,11 @@ chat_names: %w[chat_id team_id user_id], chat_teams: %w[team_id], ci_builds: %w[erased_by_id runner_id trigger_request_id user_id], + ci_namespace_mirrors: %w[namespace_id], ci_namespace_monthly_usages: %w[namespace_id], ci_pipelines: %w[user_id], ci_pipeline_chat_data: %w[chat_name_id], # it uses the loose foreign key featue + ci_project_mirrors: %w[namespace_id project_id], ci_runner_projects: %w[runner_id], ci_trigger_requests: %w[commit_id], cluster_providers_aws: %w[security_group_id vpc_id access_key_id], @@ -72,6 +74,7 @@ product_analytics_events_experimental: %w[event_id txn_id user_id], project_group_links: %w[group_id], project_statistics: %w[namespace_id], + projects_sync_events: %w[namespace_id], projects: %w[creator_id ci_id mirror_user_id], redirect_routes: %w[source_id], repository_languages: %w[programming_language_id], diff --git a/spec/lib/gitlab/background_migration/backfill_ci_namespace_mirrors_spec.rb b/spec/lib/gitlab/background_migration/backfill_ci_namespace_mirrors_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..493f8e543702b4e37da255da5b9f02b3111856f1 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_ci_namespace_mirrors_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillCiNamespaceMirrors, :migration, schema: 20211011141240 do + let(:namespaces) { table(:namespaces) } + let(:ci_namespace_mirrors) { table(:ci_namespace_mirrors) } + + subject { described_class.new } + + describe '#perform' do + it 'creates hierarchies for all namespaces in range' do + namespaces.create!(id: 5, name: 'test1', path: 'test1') + namespaces.create!(id: 7, name: 'test2', path: 'test2') + namespaces.create!(id: 8, name: 'test3', path: 'test3') + + subject.perform(5, 7, 1) + + expect(ci_namespace_mirrors.all).to contain_exactly( + an_object_having_attributes(namespace_id: 5, traversal_ids: [5]), + an_object_having_attributes(namespace_id: 7, traversal_ids: [7]) + ) + end + + it 'handles existing hierarchies gracefully' do + namespaces.create!(id: 5, name: 'test1', path: 'test1') + namespaces.create!(id: 7, name: 'test2', path: 'test2') + namespaces.create!(id: 8, name: 'test3', path: 'test3') + + # Simulate a situation where a user has had a chance to move a group to another parent + # before the background migration has had a chance to run + ci_namespace_mirrors.create!(namespace_id: 7, traversal_ids: [5, 7]) + + subject.perform(5, 7, 1) + + expect(ci_namespace_mirrors.all).to contain_exactly( + an_object_having_attributes(namespace_id: 5, traversal_ids: [5]), + an_object_having_attributes(namespace_id: 7, traversal_ids: [5, 7]) + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_ci_project_mirrors_spec.rb b/spec/lib/gitlab/background_migration/backfill_ci_project_mirrors_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..8b2138dea3f86fd8618a20b368a5f1a2fcc70e36 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_ci_project_mirrors_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillCiProjectMirrors, :migration, schema: 20211011141241 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:ci_project_mirrors) { table(:ci_project_mirrors) } + + subject { described_class.new } + + describe '#perform' do + it 'creates ci_project_mirrors for all projects in range' do + namespaces.create!(id: 10, name: 'namespace1', path: 'namespace1') + projects.create!(id: 5, namespace_id: 10, name: 'test1', path: 'test1') + projects.create!(id: 7, namespace_id: 10, name: 'test2', path: 'test2') + projects.create!(id: 8, namespace_id: 10, name: 'test3', path: 'test3') + + subject.perform(5, 7, 1) + + expect(ci_project_mirrors.all).to contain_exactly( + an_object_having_attributes(project_id: 5, namespace_id: 10), + an_object_having_attributes(project_id: 7, namespace_id: 10) + ) + end + + it 'handles existing ci_project_mirrors gracefully' do + namespaces.create!(id: 10, name: 'namespace1', path: 'namespace1') + namespaces.create!(id: 11, name: 'namespace2', path: 'namespace2', parent_id: 10) + projects.create!(id: 5, namespace_id: 10, name: 'test1', path: 'test1') + projects.create!(id: 7, namespace_id: 11, name: 'test2', path: 'test2') + projects.create!(id: 8, namespace_id: 11, name: 'test3', path: 'test3') + + # Simulate a situation where a user has had a chance to move a project to another namespace + # before the background migration has had a chance to run + ci_project_mirrors.create!(project_id: 7, namespace_id: 10) + + subject.perform(5, 7, 1) + + expect(ci_project_mirrors.all).to contain_exactly( + an_object_having_attributes(project_id: 5, namespace_id: 10), + an_object_having_attributes(project_id: 7, namespace_id: 10) + ) + end + end +end diff --git a/spec/models/ci/namespace_mirror_spec.rb b/spec/models/ci/namespace_mirror_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..75092d64c460cc54aa3e33bf1d16b8285e8c7eb7 --- /dev/null +++ b/spec/models/ci/namespace_mirror_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::NamespaceMirror do + let_it_be(:group1) { create(:group) } + let_it_be(:group2) { create(:group) } + let_it_be(:group3) { create(:group) } + let_it_be(:group4) { create(:group) } + let_it_be(:project1) { create(:project) } + let_it_be(:project2) { create(:project) } + + describe '.sync!' do + let(:event) do + double(namespace_id: namespace.id, traversal_ids: traversal_ids) + end + + subject(:sync) { described_class.sync!(event) } + + context 'when namespace hierarchy does not exist in the first place' do + let(:namespace) { group1 } + let(:traversal_ids) { [group3, group2, group1].map(&:id) } + + it 'creates the hierarchy' do + expect { sync }.to change { described_class.count }.from(0).to(1) + + expect(described_class.find(group1.id)).to have_attributes(traversal_ids: traversal_ids) + end + end + + context 'when namespace hierarchy does already exist' do + let(:namespace) { group1 } + let(:traversal_ids) { [group3, group2, group1].map(&:id) } + + before do + described_class.create!( + namespace_id: group1.id, traversal_ids: [group2, group1].map(&:id) + ) + end + + it 'updates the hierarchy' do + expect { sync }.not_to change { described_class.count } + + expect(described_class.find(group1.id)).to have_attributes(traversal_ids: traversal_ids) + end + end + + context 'changing the middle namespace' do + let(:namespace) { group2 } + let(:traversal_ids) { [group4, group3, group2].map(&:id) } + + before do + described_class.create!( + namespace_id: group1.id, traversal_ids: [group3, group2, group1].map(&:id) + ) + described_class.create!( + namespace_id: group2.id, traversal_ids: [group3, group2].map(&:id) + ) + end + + it 'updates hierarchies for the base and descendants' do + expect { sync }.not_to change { described_class.count } + + expect(described_class.find(group1.id)).to have_attributes(traversal_ids: [group4, group3, group2, group1].map(&:id)) + expect(described_class.find(group2.id)).to have_attributes(traversal_ids: [group4, group3, group2].map(&:id)) + end + end + end +end diff --git a/spec/models/ci/project_mirror_spec.rb b/spec/models/ci/project_mirror_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..2f5865f2fc2c1adaff8e6bd0af12d2451f101658 --- /dev/null +++ b/spec/models/ci/project_mirror_spec.rb @@ -0,0 +1,36 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ProjectMirror do + let_it_be(:group1) { create(:group) } + let_it_be(:group2) { create(:group) } + let_it_be(:group3) { create(:group) } + let_it_be(:project) { create(:project) } + + describe '.sync!' do + let(:event) { double(project_id: project.id, namespace_id: group2.id) } + + subject(:sync) { described_class.sync!(event) } + + context 'when project hierarchy does not exist in the first place' do + it 'creates a ci_projects record' do + expect { sync }.to change { described_class.count }.from(0).to(1) + + expect(described_class.find(project.id)).to have_attributes(namespace_id: group2.id) + end + end + + context 'when project hierarchy does already exist' do + before do + described_class.create!(project_id: project.id, namespace_id: group1.id) + end + + it 'updates the related ci_projects record' do + expect { sync }.not_to change { described_class.count } + + expect(described_class.find(project.id)).to have_attributes(namespace_id: group2.id) + end + end + end +end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 9138bb0717b701958c64360eb0d5a112bdecd4d3..69586bbea51114e3a832cc8bfef10add8c16415b 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -620,6 +620,11 @@ def stub_redis_runner_contacted_at(value) let(:group) { create(:group, projects: [build.project]) } let(:runner) { create(:ci_runner, :group, tag_list: tag_list, run_untagged: run_untagged, groups: [group]) } + before do + # Ensure the project namespace is reloaded + build.project.reload + end + it 'can handle builds' do expect(runner.can_pick?(build)).to be_truthy end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 5adbcd0fa52deabef8403c7d11c1d02a46989e6e..50649aeb00c7ff20c7c39828a3999b850d64e61f 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -2084,4 +2084,57 @@ def project_rugged(project) it { is_expected.to be(true) } end end + + describe 'namespace syncing' do + subject(:group1) { create(:group, parent: group2) } + + let_it_be(:group2) { create(:group) } + let_it_be(:group3) { create(:group) } + + it 'creates a matching namespaces_sync_event record for record without parent' do + expect(Namespaces::SyncEvent.where(namespace_id: group2.id)).to contain_exactly( + an_object_having_attributes( traversal_ids: [group2.id] ) + ) + end + + it 'creates a matching namespaces_sync_event record for record with parent' do + expect(Namespaces::SyncEvent.where(namespace_id: group1.id)).to contain_exactly( + an_object_having_attributes( traversal_ids: [group2.id, group1.id] ) + ) + end + + context 'when update to new parent is saved' do + it 'creates a matching namespaces_sync_event record' do + group1.update!(parent_id: group3.id) + + expect(Namespaces::SyncEvent.where(namespace_id: group1.id)).to contain_exactly( + an_object_having_attributes(traversal_ids: [group2.id, group1.id]), + an_object_having_attributes(traversal_ids: [group3.id, group1.id]) + ) + end + end + + context 'when update to new parent is saved twice in same transaction' do + it 'creates two namespaces_sync_event records' do + intermediate_group = create(:group) + Namespace.transaction do + group1.update!(parent_id: intermediate_group.id) + group1.update!(parent_id: group3.id) + end + + expect(Namespaces::SyncEvent.where(namespace_id: group1.id)).to contain_exactly( + an_object_having_attributes(traversal_ids: [group2.id, group1.id]), + an_object_having_attributes(traversal_ids: [intermediate_group.id, group1.id]), + an_object_having_attributes(traversal_ids: [group3.id, group1.id]) + ) + end + end + + it 'enqueues ProcessNamespaceSyncEventsWorker' do + group1 + expect(Ci::ProcessNamespaceSyncEventsWorker).to receive(:perform_async).once + + group1.update!(parent_id: group3.id) + end + end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index ec26bc5edfcb54cc54d5e70a120e83829964b628..025bc6bef06ff34a15343805b0f812c067f087c8 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -7218,7 +7218,7 @@ def has_external_wiki end describe '#enabled_group_deploy_keys' do - let_it_be(:project) { create(:project) } + let(:project) { create(:project) } subject { project.enabled_group_deploy_keys } @@ -7238,7 +7238,6 @@ def has_external_wiki let(:super_group) { create(:group) } it 'returns both group deploy keys' do - super_group = create(:group) super_group_deploy_key = create(:group_deploy_key, groups: [super_group]) group.update!(parent: super_group) @@ -7248,7 +7247,7 @@ def has_external_wiki end context 'and another group has a group deploy key enabled' do - let_it_be(:group_deploy_key) { create(:group_deploy_key) } + let(:group_deploy_key) { create(:group_deploy_key) } it 'does not return this group deploy key' do another_group = create(:group) @@ -7436,6 +7435,55 @@ def has_external_wiki end end + describe 'updating parent namespace' do + subject(:project) { create(:project) } + + let!(:original_namespace) { project.namespace } + + let_it_be(:new_parent_namespace) { create(:namespace) } + + let(:project_sync_events) { Projects::SyncEvent.where(project_id: project.id) } + + it 'creates a matching projects_sync_event record' do + expect(Projects::SyncEvent.find_by(project_id: project.id)).to have_attributes({ + project_id: project.id, namespace_id: original_namespace.id + }) + + project.update!(namespace_id: new_parent_namespace.id) + + expect(project_sync_events).to match( + [ + an_object_having_attributes(project_id: project.id, namespace_id: original_namespace.id), + an_object_having_attributes(project_id: project.id, namespace_id: new_parent_namespace.id) + ] + ) + end + + context 'twice in the same transaction' do + it 'creates two update projects_sync_event records' do + intermediate_namespace = create(:namespace) + Project.transaction do + project.update!(namespace_id: intermediate_namespace.id) + project.update!(namespace_id: new_parent_namespace.id) + end + + expect(project_sync_events).to match([ + an_object_having_attributes(project_id: project.id, namespace_id: original_namespace.id), + an_object_having_attributes(project_id: project.id, namespace_id: intermediate_namespace.id), + an_object_having_attributes(project_id: project.id, namespace_id: new_parent_namespace.id) + ]) + end + end + + it 'enqueues ProcessProjectSyncEventsWorker' do + expect(Ci::ProcessProjectSyncEventsWorker).to receive(:perform_async) + + project.update!(namespace_id: new_parent_namespace.id) + end + end + + private + def finish_job(export_job) export_job.start export_job.finish diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 8c91ee2d29f0d9bbfc368357b0d1f6af005a7144..d53eed92d165a78f12ed0c96f4c1ef1025cd9353 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1540,7 +1540,8 @@ allow(user).to receive(:update_highest_role) end - expect(SecureRandom).to receive(:hex).and_return('3b8ca303') + allow(SecureRandom).to receive(:hex).and_call_original + expect(SecureRandom).to receive(:hex).with(no_args).and_return('3b8ca303') user = create(:user) diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index 097d374640c66b9887dadfc814ea26998b864d32..3ed08afd57db4d201dab5e0ff792e4bbc82dcdd8 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -47,7 +47,7 @@ it 'executes a limited number of queries' do control_count = ActiveRecord::QueryRecorder.new { subject }.count - expect(control_count).to be <= 101 + expect(control_count).to be <= 104 end it 'schedules an import using a namespace' do diff --git a/spec/services/ci/process_sync_events_service_spec.rb b/spec/services/ci/process_sync_events_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..26fd553ae8f068e7afe762342a6de0dffb2cc4d0 --- /dev/null +++ b/spec/services/ci/process_sync_events_service_spec.rb @@ -0,0 +1,60 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ProcessSyncEventsService do + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, group: group) } + let_it_be(:parent_group_1) { create(:group) } + let_it_be(:parent_group_2) { create(:group) } + + subject(:service) { described_class.new(sync_event_class, hierarchy_class) } + + describe '#perform' do + subject(:execute) { service.execute } + + context 'for Projects::SyncEvent' do + let(:sync_event_class) { Projects::SyncEvent } + let(:hierarchy_class) { ::Ci::ProjectMirror } + + before do + Projects::SyncEvent.delete_all + Projects::SyncEvent.insert_all! [ + { project_id: project.id, namespace_id: group.id }, + { project_id: project.id, namespace_id: parent_group_1.id }, + { project_id: project.id, namespace_id: parent_group_2.id } + ] + end + + it 'consumes events' do + expect { execute }.to change(Projects::SyncEvent, :count).from(3).to(0) + + expect(Ci::ProjectMirror.where(project_id: project.id).to_a).to contain_exactly( + an_object_having_attributes(namespace_id: parent_group_2.id) + ) + end + end + + context 'for Namespaces::SyncEvent' do + let(:sync_event_class) { Namespaces::SyncEvent } + let(:hierarchy_class) { ::Ci::NamespaceMirror } + + before do + Namespaces::SyncEvent.delete_all + Namespaces::SyncEvent.insert_all! [ + { namespace_id: group.id, traversal_ids: [parent_group_1.id, group.id] }, + { namespace_id: group.id, traversal_ids: [parent_group_2.id, group.id] }, + { namespace_id: group.id, traversal_ids: [parent_group_1.id, parent_group_2.id, group.id] } + ] + end + + it 'consumes events' do + expect { execute }.to change(Namespaces::SyncEvent, :count).from(3).to(0) + + expect(Ci::NamespaceMirror.where(namespace_id: group.id).to_a).to contain_exactly( + an_object_having_attributes(traversal_ids: [parent_group_1.id, parent_group_2.id, group.id]) + ) + end + end + end +end diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index fbf5b1833658ad892ecf2e0c5826c3a391b13f94..e8f14c723aa05dffbf5f73049563716021636bec 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -2942,7 +2942,7 @@ def create_member! describe 'Pipelines', :deliver_mails_inline do describe '#pipeline_finished' do - let_it_be(:project) { create(:project, :public, :repository) } + let_it_be_with_reload(:project) { create(:project, :public, :repository) } let_it_be(:u_member) { create(:user) } let_it_be(:u_watcher) { create_user_with_notification(:watch, 'watcher') } @@ -3088,6 +3088,7 @@ def create_pipeline(user, status) let(:group_notification_email) { 'user+group@example.com' } before do + project.reload group = create(:group) project.update!(group: group) @@ -3551,6 +3552,7 @@ def build_team(project) # with different notification settings def build_group(project, visibility: :public) group = create_nested_group(visibility) + project.reload project.update!(namespace_id: group.id) # Group member: global=disabled, group=watch diff --git a/spec/workers/ci/process_namespace_sync_events_worker_spec.rb b/spec/workers/ci/process_namespace_sync_events_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..6912f2f1d02e9f4f34b572215b31e47140836cc0 --- /dev/null +++ b/spec/workers/ci/process_namespace_sync_events_worker_spec.rb @@ -0,0 +1,35 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ProcessNamespaceSyncEventsWorker do + let_it_be(:group) { create(:group) } + let_it_be(:parent_group_1) { create(:group) } + let_it_be(:parent_group_2) { create(:group) } + + include_examples 'an idempotent worker' + + describe '#perform' do + subject(:perform) { described_class.new.perform } + + before do + Namespaces::SyncEvent.insert_all! [ + { namespace_id: group.id, traversal_ids: [parent_group_1.id, group.id] }, + { namespace_id: group.id, traversal_ids: [parent_group_2.id, group.id] }, + { namespace_id: parent_group_2.id, traversal_ids: [parent_group_1.id, parent_group_2.id] } + ] + end + + it 'consumes all sync events' do + expect { perform }.to change(Namespaces::SyncEvent, :count).from(6).to(0) + end + + it 'syncs namespace hierarchy traversal ids' do + expect { perform }.to change(Ci::NamespaceMirror, :all).to contain_exactly( + an_object_having_attributes(namespace_id: parent_group_1.id, traversal_ids: [parent_group_1.id]), + an_object_having_attributes(namespace_id: parent_group_2.id, traversal_ids: [parent_group_1.id, parent_group_2.id]), + an_object_having_attributes(namespace_id: group.id, traversal_ids: [parent_group_1.id, parent_group_2.id, group.id]) + ) + end + end +end diff --git a/spec/workers/ci/process_project_sync_events_worker_spec.rb b/spec/workers/ci/process_project_sync_events_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..6b3beb0b998e489b142d094d086281f2251bd450 --- /dev/null +++ b/spec/workers/ci/process_project_sync_events_worker_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ProcessProjectSyncEventsWorker do + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, group: group) } + let_it_be(:parent_group_1) { create(:group) } + let_it_be(:parent_group_2) { create(:group) } + + include_examples 'an idempotent worker' + + describe '#perform' do + subject(:perform) { described_class.new.perform } + + before do + Projects::SyncEvent.insert_all! [ + { project_id: project.id, namespace_id: parent_group_1.id }, + { project_id: project.id, namespace_id: parent_group_2.id } + ] + end + + it 'consumes all sync events' do + expect { perform }.to change(Projects::SyncEvent, :count).from(3).to(0) + end + + it 'syncs project hierarchy traversal ids' do + expect { perform }.to change(Ci::ProjectMirror.where(project_id: project.id), :all).to contain_exactly( + an_object_having_attributes(namespace_id: parent_group_2.id) + ) + end + end +end