diff --git a/app/models/ci/namespace_hierarchy.rb b/app/models/ci/namespace_hierarchy.rb new file mode 100644 index 0000000000000000000000000000000000000000..14482b9ea5ad8ea2f749736d653226b2b0c95acc --- /dev/null +++ b/app/models/ci/namespace_hierarchy.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module Ci + # This model represents a record in a shadow table of the main database's namespaces table. + # It allows us to navigate the namespace hierarchy on the ci database without resorting to a JOIN. + class NamespaceHierarchy < ApplicationRecord + class << self + def update_traversal_ids(old_self_and_ancestor_ids, new_self_and_ancestor_ids) + # Update the traversal IDs with the new_self_and_ancestor_ids plus any descendants in the record + where('traversal_ids @> ARRAY[?]::int[]', old_self_and_ancestor_ids) + .update_all("traversal_ids = ARRAY[#{new_self_and_ancestor_ids.join(',')}]::int[] || traversal_ids[#{old_self_and_ancestor_ids.length + 1}:]") + end + + def sync_traversal_ids(event) + if event.traversal_ids.present? + update_traversal_ids(event.traversal_ids, event.new_traversal_ids) + ::Ci::ProjectHierarchy.update_namespace_traversal_ids(event.traversal_ids, event.new_traversal_ids) + else + create!(namespace_id: event.namespace_id, traversal_ids: event.new_traversal_ids) + end + end + end + end +end diff --git a/app/models/ci/project_hierarchy.rb b/app/models/ci/project_hierarchy.rb new file mode 100644 index 0000000000000000000000000000000000000000..392ef9a0b45cb4d1e62d530eb83f3dae6370ab8d --- /dev/null +++ b/app/models/ci/project_hierarchy.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +module Ci + # This model represents a shadow table of the main database's namespaces table. + # It allows us to navigate the namespace hierarchy on the ci database for a project without resorting to a JOIN. + class ProjectHierarchy < ApplicationRecord + belongs_to :project, class_name: 'Project' + + class << self + def update_traversal_ids(project_id, new_ancestor_ids) + where(project_id: project_id).update_all(traversal_ids: new_ancestor_ids) + end + + def update_namespace_traversal_ids(old_ancestor_ids, new_ancestor_ids) + update_all_traversal_ids(where('traversal_ids @> ARRAY[?]::int[]', old_ancestor_ids), old_ancestor_ids, new_ancestor_ids) + end + + def sync_traversal_ids(event) + if event.traversal_ids.present? + update_traversal_ids(event.project_id, event.new_traversal_ids) + else + create!(project_id: event.project_id, traversal_ids: event.new_traversal_ids) + end + end + + private + + def update_all_traversal_ids(subject, old_ancestor_ids, new_ancestor_ids) + # Update the traversal IDs with the new_ancestor_ids plus any descendants in the record + subject.update_all("traversal_ids = ARRAY[#{new_ancestor_ids.join(',')}]::int[] || traversal_ids[#{old_ancestor_ids.length + 1}:]") + end + end + end +end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 9565d1a6a692bc87e3842cfa1a16be143ebb6cc7..d7ade8791b714a22a8a90c329360cc31e41018ae 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -107,6 +107,8 @@ class Namespace < ApplicationRecord delegate :name, to: :owner, allow_nil: true, prefix: true delegate :avatar_url, to: :owner, allow_nil: true + after_save :schedule_ci_hierarchy_update, if: -> { saved_change_to_id? || saved_change_to_parent_id? } + after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } before_create :sync_share_with_group_lock_with_parent @@ -616,6 +618,26 @@ def write_projects_repository_config def enforce_minimum_path_length? path_changed? && !project_namespace? end + + def schedule_ci_hierarchy_update + previous_namespace_ids = + if previously_new_record? + [] + else + previous_namespace = Namespace.find(parent_id_previously_was) if parent_id_previously_was + (previous_namespace&.self_and_ancestor_ids(hierarchy_order: :desc) || []) + [id] + end + + Namespaces::SyncEvent.insert({ + namespace_id: id, + traversal_ids: previous_namespace_ids, + new_traversal_ids: self_and_ancestor_ids(hierarchy_order: :desc) + }) + + run_after_commit do + Ci::ProcessNamespaceSyncEventsWorker.perform_async + end + end end Namespace.prepend_mod_with('Namespace') diff --git a/app/models/namespaces/sync_event.rb b/app/models/namespaces/sync_event.rb new file mode 100644 index 0000000000000000000000000000000000000000..5acd4410a89f6a3a677c5ae6ac06f8b46de60c07 --- /dev/null +++ b/app/models/namespaces/sync_event.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +# This model serves to keep track of changes to the namespaces table in the main database, and allowing to safely +# replicate these changes to other databases. +class Namespaces::SyncEvent < ApplicationRecord + self.table_name = 'namespaces_sync_events' +end diff --git a/app/models/project.rb b/app/models/project.rb index 604158d1a6eb22fd841a7c8d962b9119ddf3e78e..a6503ad1ff7a19d4a25188d1cfe730630d5eed82 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -101,6 +101,8 @@ class Project < ApplicationRecord after_save :update_project_statistics, if: :saved_change_to_namespace_id? + after_save :schedule_ci_hierarchy_update, if: -> { saved_change_to_id? || saved_change_to_namespace_id? } + after_save :create_import_state, if: ->(project) { project.import? && project.import_state.nil? } after_save :save_topics @@ -2955,6 +2957,26 @@ def sync_attributes(project_namespace) project_namespace.shared_runners_enabled = shared_runners_enabled project_namespace.visibility_level = visibility_level end + + def schedule_ci_hierarchy_update + previous_namespace_ids = + if previously_new_record? + [] + else + previous_namespace = Namespace.find(namespace_id_previously_was) if namespace_id_previously_was + previous_namespace&.self_and_ancestor_ids(hierarchy_order: :desc) || [] + end + + Projects::SyncEvent.insert({ + project_id: id, + traversal_ids: previous_namespace_ids, + new_traversal_ids: namespace.self_and_ancestor_ids(hierarchy_order: :desc) + }) + + run_after_commit do + Ci::ProcessProjectSyncEventsWorker.perform_async + end + end end Project.prepend_mod_with('Project') diff --git a/app/models/projects/sync_event.rb b/app/models/projects/sync_event.rb new file mode 100644 index 0000000000000000000000000000000000000000..548710fe255d019311523cb813d09672b773f60f --- /dev/null +++ b/app/models/projects/sync_event.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +# This model serves to keep track of changes to the namespaces table in the main database as they relate to projects, +# allowing to safely replicate changes to other databases. +class Projects::SyncEvent < ApplicationRecord + self.table_name = 'projects_sync_events' +end diff --git a/app/services/ci/process_hierarchy_sync_events_service.rb b/app/services/ci/process_hierarchy_sync_events_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..05f48411aecd06618f3f97e51174c6e1ce06504d --- /dev/null +++ b/app/services/ci/process_hierarchy_sync_events_service.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Ci + class ProcessHierarchySyncEventsService + BATCH_SIZE = 100 + + def initialize(sync_event_class, hierarchy_class) + @sync_event_class = sync_event_class + @hierarchy_class = hierarchy_class + end + + def execute + # rubocop: disable CodeReuse/ActiveRecord + @sync_event_class.find_in_batches(batch_size: BATCH_SIZE) do |batch| + batch.each { |evt| @hierarchy_class.sync_traversal_ids(evt) if evt.new_traversal_ids.present? } + + select_query = @sync_event_class + .id_in(batch) + .lock!('FOR UPDATE SKIP LOCKED') + .select(:id) + + @sync_event_class.where(id: select_query).delete_all + end + # rubocop: enable CodeReuse/ActiveRecord + end + end +end diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index 4fa74195a993bb420f91bed11cfa906e057ea012..f8fef9061e52e96cc9ad5688e11165b28b20f925 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -1951,6 +1951,24 @@ :weight: 1 :idempotent: true :tags: [] +- :name: ci_process_namespace_sync_events + :worker_name: Ci::ProcessNamespaceSyncEventsWorker + :feature_category: :subgroups + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] +- :name: ci_process_project_sync_events + :worker_name: Ci::ProcessProjectSyncEventsWorker + :feature_category: :subgroups + :has_external_dependencies: + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: create_commit_signature :worker_name: CreateCommitSignatureWorker :feature_category: :source_code_management diff --git a/app/workers/ci/process_namespace_sync_events_worker.rb b/app/workers/ci/process_namespace_sync_events_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..0eedab208bdfa277c81089e9af78e091f3bde12b --- /dev/null +++ b/app/workers/ci/process_namespace_sync_events_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Ci + class ProcessNamespaceSyncEventsWorker + include ApplicationWorker + + data_consistency :always + + feature_category :subgroups + + deduplicate :until_executed + idempotent! + + def perform + Ci::ProcessHierarchySyncEventsService.new(Namespaces::SyncEvent, ::Ci::NamespaceHierarchy).execute + end + end +end diff --git a/app/workers/ci/process_project_sync_events_worker.rb b/app/workers/ci/process_project_sync_events_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..1e40e1ae3439678dc43f0541b5d39aa1200a2562 --- /dev/null +++ b/app/workers/ci/process_project_sync_events_worker.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +module Ci + class ProcessProjectSyncEventsWorker + include ApplicationWorker + + data_consistency :always + + feature_category :subgroups + + deduplicate :until_executed + idempotent! + + def perform + Ci::ProcessHierarchySyncEventsService.new(Projects::SyncEvent, ::Ci::ProjectHierarchy).execute + end + end +end diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index f7e3f036c53547e2c8dd99d8bd0faf0c02d51f56..77c901c3d094752af05172dd87730036da2bbc43 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -73,6 +73,10 @@ - 1 - - ci_delete_objects - 1 +- - ci_process_namespace_sync_events + - 1 +- - ci_process_project_sync_events + - 1 - - container_repository - 1 - - create_commit_signature diff --git a/db/migrate/20211011140930_create_ci_namespace_hierarchies.rb b/db/migrate/20211011140930_create_ci_namespace_hierarchies.rb new file mode 100644 index 0000000000000000000000000000000000000000..a0f1999dc7a82e0415b4311635bdd02694107d85 --- /dev/null +++ b/db/migrate/20211011140930_create_ci_namespace_hierarchies.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CreateCiNamespaceHierarchies < Gitlab::Database::Migration[1.0] + TABLE_NAME = :ci_namespace_hierarchies + INDEX_NAME = "index_gin_#{TABLE_NAME}" + + def change + create_table TABLE_NAME, id: false do |t| + t.references :namespace, primary_key: true, null: false, index: false, foreign_key: { on_delete: :cascade } + t.integer :traversal_ids, array: true, default: [], null: false + + t.index :traversal_ids, name: INDEX_NAME, using: :gin + end + end +end diff --git a/db/migrate/20211011140931_create_ci_project_hierarchies.rb b/db/migrate/20211011140931_create_ci_project_hierarchies.rb new file mode 100644 index 0000000000000000000000000000000000000000..595e914ec7112862f7472db9e151023dc8b34545 --- /dev/null +++ b/db/migrate/20211011140931_create_ci_project_hierarchies.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class CreateCiProjectHierarchies < Gitlab::Database::Migration[1.0] + TABLE_NAME = :ci_project_hierarchies + INDEX_NAME = "index_gin_#{TABLE_NAME}" + + def change + create_table TABLE_NAME, id: false do |t| + t.references :project, primary_key: true, null: false, index: false, foreign_key: { on_delete: :cascade } + t.integer :traversal_ids, array: true, default: [], null: false + + t.index :traversal_ids, name: INDEX_NAME, using: :gin + end + end +end diff --git a/db/migrate/20211011140932_create_namespaces_sync_events.rb b/db/migrate/20211011140932_create_namespaces_sync_events.rb new file mode 100644 index 0000000000000000000000000000000000000000..bf2dbb41623bd9f8c782a517b4aac62549b79794 --- /dev/null +++ b/db/migrate/20211011140932_create_namespaces_sync_events.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class CreateNamespacesSyncEvents < Gitlab::Database::Migration[1.0] + def change + create_table :namespaces_sync_events do |t| + t.references :namespace, null: false, index: true, foreign_key: { on_delete: :cascade } + t.integer :traversal_ids, array: true, default: [], null: false + t.integer :new_traversal_ids, array: true, default: [], null: false + end + end +end diff --git a/db/migrate/20211011141239_create_projects_sync_events.rb b/db/migrate/20211011141239_create_projects_sync_events.rb new file mode 100644 index 0000000000000000000000000000000000000000..be1c89ca5703508852843dbbd2bbf247dc93f57b --- /dev/null +++ b/db/migrate/20211011141239_create_projects_sync_events.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +class CreateProjectsSyncEvents < Gitlab::Database::Migration[1.0] + def change + create_table :projects_sync_events do |t| + t.references :project, null: false, index: true, foreign_key: { on_delete: :cascade } + t.integer :traversal_ids, array: true, default: [], null: false + t.integer :new_traversal_ids, array: true, default: [], null: false + end + end +end diff --git a/db/post_migrate/20211011141240_add_ci_namespace_hierarchies.rb b/db/post_migrate/20211011141240_add_ci_namespace_hierarchies.rb new file mode 100644 index 0000000000000000000000000000000000000000..54acc0fe9d335d31e377d202ee8abdc480b3116a --- /dev/null +++ b/db/post_migrate/20211011141240_add_ci_namespace_hierarchies.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddCiNamespaceHierarchies < Gitlab::Database::Migration[1.0] + MIGRATION = 'BackfillNamespaceHierarchies' + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 100 + DELAY_INTERVAL = 2.minutes + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::BackfillNamespaceHierarchies::Namespace.base_query, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + other_job_arguments: [SUB_BATCH_SIZE], + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/post_migrate/20211011141241_add_ci_project_hierarchies.rb b/db/post_migrate/20211011141241_add_ci_project_hierarchies.rb new file mode 100644 index 0000000000000000000000000000000000000000..c37e15d8f968c750ef11aac73981978192fcd44b --- /dev/null +++ b/db/post_migrate/20211011141241_add_ci_project_hierarchies.rb @@ -0,0 +1,28 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddCiProjectHierarchies < Gitlab::Database::Migration[1.0] + MIGRATION = 'BackfillProjectHierarchies' + BATCH_SIZE = 1_000 + SUB_BATCH_SIZE = 100 + DELAY_INTERVAL = 2.minutes + + disable_ddl_transaction! + + def up + queue_background_migration_jobs_by_range_at_intervals( + Gitlab::BackgroundMigration::BackfillProjectHierarchies::Project.base_query, + MIGRATION, + DELAY_INTERVAL, + batch_size: BATCH_SIZE, + other_job_arguments: [SUB_BATCH_SIZE], + track_jobs: true + ) + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20211011140930 b/db/schema_migrations/20211011140930 new file mode 100644 index 0000000000000000000000000000000000000000..6347ee5d51d84d730d8203eb87c586c2e46f1d24 --- /dev/null +++ b/db/schema_migrations/20211011140930 @@ -0,0 +1 @@ +cdae819e8de3b5ad721014376bfd9af97a45e953e2d345daf62784f986a5eb31 \ No newline at end of file diff --git a/db/schema_migrations/20211011140931 b/db/schema_migrations/20211011140931 new file mode 100644 index 0000000000000000000000000000000000000000..c959d97074e02d7daf41a54dfe12cbe8d3d2bdf3 --- /dev/null +++ b/db/schema_migrations/20211011140931 @@ -0,0 +1 @@ +7e51eb4443fd74da9bef4d9c1c3cc40376c311abbc05ca7871f725fada79b48a \ No newline at end of file diff --git a/db/schema_migrations/20211011140932 b/db/schema_migrations/20211011140932 new file mode 100644 index 0000000000000000000000000000000000000000..af0e000b9f3e2a5512c10c788ec787df1596e86c --- /dev/null +++ b/db/schema_migrations/20211011140932 @@ -0,0 +1 @@ +0209db1e7be48bcbf0e52b451d37da0ef2ecadd567cdfa47907fc5032c258a27 \ No newline at end of file diff --git a/db/schema_migrations/20211011141239 b/db/schema_migrations/20211011141239 new file mode 100644 index 0000000000000000000000000000000000000000..f215f234a7e1a5528b28235bb5dec434dc7ef763 --- /dev/null +++ b/db/schema_migrations/20211011141239 @@ -0,0 +1 @@ +bc0ae055b331801fbe020c12a66e4e6ae790780121bfd66fd161093c94c7a84a \ No newline at end of file diff --git a/db/schema_migrations/20211011141240 b/db/schema_migrations/20211011141240 new file mode 100644 index 0000000000000000000000000000000000000000..ee736c5e0267cf75a5cf5826ad54f1f0448c3106 --- /dev/null +++ b/db/schema_migrations/20211011141240 @@ -0,0 +1 @@ +f5dad108d30fe1d3a36027cd3f6fd7db006444892e5ec145921117d490be11c7 \ No newline at end of file diff --git a/db/schema_migrations/20211011141241 b/db/schema_migrations/20211011141241 new file mode 100644 index 0000000000000000000000000000000000000000..33c6386425da192feb59f9255f94e4ab9e1051ac --- /dev/null +++ b/db/schema_migrations/20211011141241 @@ -0,0 +1 @@ +347aa02789f2016271cd669de005bb722283a4c59e48a2ca831342e123e38b27 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 7e366cd67d3a6b25c321293075a07e7545dfdbb6..12988dec77e7dd838249591628ff29d15ec3ae90 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11740,6 +11740,20 @@ CREATE SEQUENCE ci_minutes_additional_packs_id_seq ALTER SEQUENCE ci_minutes_additional_packs_id_seq OWNED BY ci_minutes_additional_packs.id; +CREATE TABLE ci_namespace_hierarchies ( + namespace_id bigint NOT NULL, + traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL +); + +CREATE SEQUENCE ci_namespace_hierarchies_namespace_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ci_namespace_hierarchies_namespace_id_seq OWNED BY ci_namespace_hierarchies.namespace_id; + CREATE TABLE ci_namespace_monthly_usages ( id bigint NOT NULL, namespace_id bigint NOT NULL, @@ -11987,6 +12001,20 @@ CREATE SEQUENCE ci_platform_metrics_id_seq ALTER SEQUENCE ci_platform_metrics_id_seq OWNED BY ci_platform_metrics.id; +CREATE TABLE ci_project_hierarchies ( + project_id bigint NOT NULL, + traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL +); + +CREATE SEQUENCE ci_project_hierarchies_project_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ci_project_hierarchies_project_id_seq OWNED BY ci_project_hierarchies.project_id; + CREATE TABLE ci_project_monthly_usages ( id bigint NOT NULL, project_id bigint NOT NULL, @@ -16365,6 +16393,22 @@ CREATE SEQUENCE namespaces_id_seq ALTER SEQUENCE namespaces_id_seq OWNED BY namespaces.id; +CREATE TABLE namespaces_sync_events ( + id bigint NOT NULL, + namespace_id bigint NOT NULL, + traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL, + new_traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL +); + +CREATE SEQUENCE namespaces_sync_events_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE namespaces_sync_events_id_seq OWNED BY namespaces_sync_events.id; + CREATE TABLE note_diff_files ( id integer NOT NULL, diff_note_id integer NOT NULL, @@ -18441,6 +18485,22 @@ CREATE SEQUENCE projects_id_seq ALTER SEQUENCE projects_id_seq OWNED BY projects.id; +CREATE TABLE projects_sync_events ( + id bigint NOT NULL, + project_id bigint NOT NULL, + traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL, + new_traversal_ids integer[] DEFAULT '{}'::integer[] NOT NULL +); + +CREATE SEQUENCE projects_sync_events_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE projects_sync_events_id_seq OWNED BY projects_sync_events.id; + CREATE TABLE prometheus_alert_events ( id bigint NOT NULL, project_id integer NOT NULL, @@ -21239,6 +21299,8 @@ ALTER TABLE ONLY ci_job_variables ALTER COLUMN id SET DEFAULT nextval('ci_job_va ALTER TABLE ONLY ci_minutes_additional_packs ALTER COLUMN id SET DEFAULT nextval('ci_minutes_additional_packs_id_seq'::regclass); +ALTER TABLE ONLY ci_namespace_hierarchies ALTER COLUMN namespace_id SET DEFAULT nextval('ci_namespace_hierarchies_namespace_id_seq'::regclass); + ALTER TABLE ONLY ci_namespace_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_namespace_monthly_usages_id_seq'::regclass); ALTER TABLE ONLY ci_pending_builds ALTER COLUMN id SET DEFAULT nextval('ci_pending_builds_id_seq'::regclass); @@ -21261,6 +21323,8 @@ ALTER TABLE ONLY ci_pipelines_config ALTER COLUMN pipeline_id SET DEFAULT nextva ALTER TABLE ONLY ci_platform_metrics ALTER COLUMN id SET DEFAULT nextval('ci_platform_metrics_id_seq'::regclass); +ALTER TABLE ONLY ci_project_hierarchies ALTER COLUMN project_id SET DEFAULT nextval('ci_project_hierarchies_project_id_seq'::regclass); + ALTER TABLE ONLY ci_project_monthly_usages ALTER COLUMN id SET DEFAULT nextval('ci_project_monthly_usages_id_seq'::regclass); ALTER TABLE ONLY ci_refs ALTER COLUMN id SET DEFAULT nextval('ci_refs_id_seq'::regclass); @@ -21647,6 +21711,8 @@ ALTER TABLE ONLY namespace_statistics ALTER COLUMN id SET DEFAULT nextval('names ALTER TABLE ONLY namespaces ALTER COLUMN id SET DEFAULT nextval('namespaces_id_seq'::regclass); +ALTER TABLE ONLY namespaces_sync_events ALTER COLUMN id SET DEFAULT nextval('namespaces_sync_events_id_seq'::regclass); + ALTER TABLE ONLY note_diff_files ALTER COLUMN id SET DEFAULT nextval('note_diff_files_id_seq'::regclass); ALTER TABLE ONLY notes ALTER COLUMN id SET DEFAULT nextval('notes_id_seq'::regclass); @@ -21799,6 +21865,8 @@ ALTER TABLE ONLY project_tracing_settings ALTER COLUMN id SET DEFAULT nextval('p ALTER TABLE ONLY projects ALTER COLUMN id SET DEFAULT nextval('projects_id_seq'::regclass); +ALTER TABLE ONLY projects_sync_events ALTER COLUMN id SET DEFAULT nextval('projects_sync_events_id_seq'::regclass); + ALTER TABLE ONLY prometheus_alert_events ALTER COLUMN id SET DEFAULT nextval('prometheus_alert_events_id_seq'::regclass); ALTER TABLE ONLY prometheus_alerts ALTER COLUMN id SET DEFAULT nextval('prometheus_alerts_id_seq'::regclass); @@ -22696,6 +22764,9 @@ ALTER TABLE ONLY ci_job_variables ALTER TABLE ONLY ci_minutes_additional_packs ADD CONSTRAINT ci_minutes_additional_packs_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_namespace_hierarchies + ADD CONSTRAINT ci_namespace_hierarchies_pkey PRIMARY KEY (namespace_id); + ALTER TABLE ONLY ci_namespace_monthly_usages ADD CONSTRAINT ci_namespace_monthly_usages_pkey PRIMARY KEY (id); @@ -22729,6 +22800,9 @@ ALTER TABLE ONLY ci_pipelines ALTER TABLE ONLY ci_platform_metrics ADD CONSTRAINT ci_platform_metrics_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_project_hierarchies + ADD CONSTRAINT ci_project_hierarchies_pkey PRIMARY KEY (project_id); + ALTER TABLE ONLY ci_project_monthly_usages ADD CONSTRAINT ci_project_monthly_usages_pkey PRIMARY KEY (id); @@ -23395,6 +23469,9 @@ ALTER TABLE ONLY namespace_statistics ALTER TABLE ONLY namespaces ADD CONSTRAINT namespaces_pkey PRIMARY KEY (id); +ALTER TABLE ONLY namespaces_sync_events + ADD CONSTRAINT namespaces_sync_events_pkey PRIMARY KEY (id); + ALTER TABLE ONLY note_diff_files ADD CONSTRAINT note_diff_files_pkey PRIMARY KEY (id); @@ -23668,6 +23745,9 @@ ALTER TABLE ONLY project_tracing_settings ALTER TABLE ONLY projects ADD CONSTRAINT projects_pkey PRIMARY KEY (id); +ALTER TABLE ONLY projects_sync_events + ADD CONSTRAINT projects_sync_events_pkey PRIMARY KEY (id); + ALTER TABLE ONLY prometheus_alert_events ADD CONSTRAINT prometheus_alert_events_pkey PRIMARY KEY (id); @@ -25958,8 +26038,12 @@ CREATE INDEX index_geo_repository_updated_events_on_source ON geo_repository_upd CREATE INDEX index_geo_reset_checksum_events_on_project_id ON geo_reset_checksum_events USING btree (project_id); +CREATE INDEX index_gin_ci_namespace_hierarchies ON ci_namespace_hierarchies USING gin (traversal_ids); + CREATE INDEX index_gin_ci_pending_builds_on_namespace_traversal_ids ON ci_pending_builds USING gin (namespace_traversal_ids); +CREATE INDEX index_gin_ci_project_hierarchies ON ci_project_hierarchies USING gin (traversal_ids); + CREATE INDEX index_gitlab_subscription_histories_on_gitlab_subscription_id ON gitlab_subscription_histories USING btree (gitlab_subscription_id); CREATE INDEX index_gitlab_subscriptions_on_end_date_and_namespace_id ON gitlab_subscriptions USING btree (end_date, namespace_id); @@ -26512,6 +26596,8 @@ CREATE INDEX index_namespaces_on_type_and_id ON namespaces USING btree (type, id CREATE INDEX index_namespaces_public_groups_name_id ON namespaces USING btree (name, id) WHERE (((type)::text = 'Group'::text) AND (visibility_level = 20)); +CREATE INDEX index_namespaces_sync_events_on_namespace_id ON namespaces_sync_events USING btree (namespace_id); + CREATE INDEX index_non_requested_project_members_on_source_id_and_type ON members USING btree (source_id, source_type) WHERE ((requested_at IS NULL) AND ((type)::text = 'ProjectMember'::text)); CREATE UNIQUE INDEX index_note_diff_files_on_diff_note_id ON note_diff_files USING btree (diff_note_id); @@ -26974,6 +27060,8 @@ CREATE INDEX index_projects_on_star_count ON projects USING btree (star_count); CREATE INDEX index_projects_on_updated_at_and_id ON projects USING btree (updated_at, id); +CREATE INDEX index_projects_sync_events_on_project_id ON projects_sync_events USING btree (project_id); + CREATE UNIQUE INDEX index_prometheus_alert_event_scoped_payload_key ON prometheus_alert_events USING btree (prometheus_alert_id, payload_key); CREATE INDEX index_prometheus_alert_events_on_project_id_and_status ON prometheus_alert_events USING btree (project_id, status); @@ -30382,6 +30470,9 @@ ALTER TABLE ONLY operations_scopes ALTER TABLE ONLY milestone_releases ADD CONSTRAINT fk_rails_7ae0756a2d FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_project_hierarchies + ADD CONSTRAINT fk_rails_7d69e0670e FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY resource_state_events ADD CONSTRAINT fk_rails_7ddc5f7457 FOREIGN KEY (source_merge_request_id) REFERENCES merge_requests(id) ON DELETE SET NULL; @@ -30565,6 +30656,9 @@ ALTER TABLE ONLY gpg_keys ALTER TABLE ONLY analytics_language_trend_repository_languages ADD CONSTRAINT fk_rails_9d851d566c FOREIGN KEY (programming_language_id) REFERENCES programming_languages(id) ON DELETE CASCADE; +ALTER TABLE ONLY namespaces_sync_events + ADD CONSTRAINT fk_rails_9da32a0431 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY badges ADD CONSTRAINT fk_rails_9df4a56538 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; @@ -30739,6 +30833,9 @@ ALTER TABLE ONLY security_findings ALTER TABLE ONLY packages_debian_project_component_files ADD CONSTRAINT fk_rails_bbe9ebfbd9 FOREIGN KEY (component_id) REFERENCES packages_debian_project_components(id) ON DELETE RESTRICT; +ALTER TABLE ONLY projects_sync_events + ADD CONSTRAINT fk_rails_bbf0eef59f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY approval_merge_request_rules_users ADD CONSTRAINT fk_rails_bc8972fa55 FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; @@ -30934,6 +31031,9 @@ ALTER TABLE ONLY packages_debian_group_component_files ALTER TABLE ONLY user_callouts ADD CONSTRAINT fk_rails_ddfdd80f3d FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; +ALTER TABLE ONLY ci_namespace_hierarchies + ADD CONSTRAINT fk_rails_de9e668a9e FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY vulnerability_feedback ADD CONSTRAINT fk_rails_debd54e456 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; diff --git a/ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb b/ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb index 19add5ac38449022cbc2eec701cf09a911bf6070..c74cc34d5633086c64bfb49eb7ab3cec4faf3f87 100644 --- a/ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb +++ b/ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb @@ -15,7 +15,7 @@ described_class.uncached_data end - expect(recorder.count).to eq(59) + expect(recorder.count).to eq(63) end end end diff --git a/ee/spec/models/ci/daily_build_group_report_result_spec.rb b/ee/spec/models/ci/daily_build_group_report_result_spec.rb index d622e92d963907ec78e91f88274aa6be69d8c130..e647c58516280b6ea8c1fc955c3367ca0a952299 100644 --- a/ee/spec/models/ci/daily_build_group_report_result_spec.rb +++ b/ee/spec/models/ci/daily_build_group_report_result_spec.rb @@ -100,8 +100,9 @@ end context 'when group has projects with several coverage' do - let!(:project_2) { create(:project) } - let!(:group) { create(:group, projects: [project, project_2]) } + let_it_be(:project_2) { create(:project) } + let_it_be(:group) { create(:group, projects: [project, project_2]) } + let!(:coverage_1) { create(:ci_daily_build_group_report_result, project: project) } let!(:coverage_2) { create(:ci_daily_build_group_report_result, project: project_2, group_name: 'karma') } diff --git a/lib/gitlab/background_migration/backfill_namespace_hierarchies.rb b/lib/gitlab/background_migration/backfill_namespace_hierarchies.rb new file mode 100644 index 0000000000000000000000000000000000000000..4bb44d76875666d1d140d09e76ebbff0011fd6fb --- /dev/null +++ b/lib/gitlab/background_migration/backfill_namespace_hierarchies.rb @@ -0,0 +1,75 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to create ci_namespace_hierarchies entries in sub-batches, relative to all namespaces + # that don't yet have a match in ci_namespace_hierarchies. + # rubocop:disable Style/Documentation + class BackfillNamespaceHierarchies + class Namespace < ActiveRecord::Base + include ::EachBatch + + self.table_name = 'namespaces' + self.inheritance_column = nil + + scope :base_query, -> do + select(:id, :parent_id) + end + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + batch_query = Namespace.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first + ranged_query = Namespace.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + INSERT INTO ci_namespace_hierarchies (namespace_id, traversal_ids) + #{calculated_traversal_ids(ranged_query.allow_cross_joins_across_databases(url: ''))} + ON CONFLICT (namespace_id) DO NOTHING + SQL + ActiveRecord::Base.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + # We have to add all arguments when marking a job as succeeded as they + # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + # Calculate the ancestor path for a given set of namespaces. + def calculated_traversal_ids(batch) + <<~SQL + WITH RECURSIVE cte(source_id, namespace_id, parent_id, height) AS ( + ( + SELECT batch.id, batch.id, batch.parent_id, 1 + FROM (#{batch.to_sql}) AS batch + ) + UNION ALL + ( + SELECT cte.source_id, n.id, n.parent_id, cte.height+1 + FROM namespaces n, cte + WHERE n.id = cte.parent_id + ) + ) + SELECT flat_hierarchy.source_id as namespace_id, + array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) as traversal_ids + FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy + GROUP BY flat_hierarchy.source_id + SQL + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillNamespaceHierarchies', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/background_migration/backfill_project_hierarchies.rb b/lib/gitlab/background_migration/backfill_project_hierarchies.rb new file mode 100644 index 0000000000000000000000000000000000000000..e58f708f61532dc3c730886f935ae010cff539cf --- /dev/null +++ b/lib/gitlab/background_migration/backfill_project_hierarchies.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module Gitlab + module BackgroundMigration + # A job to set ci_project_hierarchies.traversal_ids in sub-batches, of all projects + # that don't yet have a match in ci_project_hierarchies. + # rubocop:disable Style/Documentation + class BackfillProjectHierarchies + class Project < ActiveRecord::Base + include ::EachBatch + + belongs_to :namespace + + self.table_name = 'projects' + + scope :base_query, -> do + joins(:namespace).select(:id, 'namespaces.id AS parent_id') + end + end + + PAUSE_SECONDS = 0.1 + + def perform(start_id, end_id, sub_batch_size) + batch_query = Project.base_query.where(id: start_id..end_id) + batch_query.each_batch(of: sub_batch_size) do |sub_batch| + first, last = sub_batch.pluck(Arel.sql('min(projects.id), max(projects.id)')).first + ranged_query = Project.unscoped.base_query.where(id: first..last) + + update_sql = <<~SQL + INSERT INTO ci_project_hierarchies (project_id, traversal_ids) + #{calculated_traversal_ids(ranged_query.allow_cross_joins_across_databases(url: ''))} + ON CONFLICT (project_id) DO NOTHING + SQL + ActiveRecord::Base.connection.execute(update_sql) + + sleep PAUSE_SECONDS + end + + # We have to add all arguments when marking a job as succeeded as they + # are all used to track the job by `queue_background_migration_jobs_by_range_at_intervals` + mark_job_as_succeeded(start_id, end_id, sub_batch_size) + end + + private + + # Calculate the ancestor path for a given set of projects. + def calculated_traversal_ids(batch) + <<~SQL + ( + WITH RECURSIVE cte ( + source_id, + namespace_id, + parent_id, + height + ) AS (( + SELECT + batch.id, + 0, + batch.parent_id, + 1 + FROM (#{batch.to_sql}) AS batch) + UNION ALL ( + SELECT + cte.source_id, + n.id, + n.parent_id, + cte.height + 1 + FROM + namespaces n, + cte + WHERE + n.id = cte.parent_id)) + SELECT + h.project_id, + h.traversal_ids[1:array_length(h.traversal_ids, 1) - 1] + FROM ( + SELECT + flat_hierarchy.source_id AS project_id, + array_agg(flat_hierarchy.namespace_id ORDER BY flat_hierarchy.height DESC) AS traversal_ids + FROM (SELECT * FROM cte FOR UPDATE) flat_hierarchy + GROUP BY + flat_hierarchy.source_id) AS h + ) + SQL + end + + def mark_job_as_succeeded(*arguments) + Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded( + 'BackfillProjectHierarchies', + arguments + ) + end + end + end +end diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index ee5039ccab8389bc65b521130b6e53252c346fa5..ac544bff2f88f609c1aaa195c4cefe203c2698f7 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -84,6 +84,7 @@ ci_job_artifacts: :gitlab_ci ci_job_token_project_scope_links: :gitlab_ci ci_job_variables: :gitlab_ci ci_minutes_additional_packs: :gitlab_ci +ci_namespace_hierarchies: :gitlab_ci ci_namespace_monthly_usages: :gitlab_ci ci_pending_builds: :gitlab_ci ci_pipeline_artifacts: :gitlab_ci @@ -95,6 +96,7 @@ ci_pipelines_config: :gitlab_ci ci_pipelines: :gitlab_ci ci_pipeline_variables: :gitlab_ci ci_platform_metrics: :gitlab_ci +ci_project_hierarchies: :gitlab_ci ci_project_monthly_usages: :gitlab_ci ci_refs: :gitlab_ci ci_resource_groups: :gitlab_ci @@ -314,6 +316,7 @@ namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main namespace_settings: :gitlab_main namespaces: :gitlab_main +namespaces_sync_events: :gitlab_main namespace_statistics: :gitlab_main note_diff_files: :gitlab_main notes: :gitlab_main @@ -409,6 +412,7 @@ project_repository_storage_moves: :gitlab_main project_security_settings: :gitlab_main project_settings: :gitlab_main projects: :gitlab_main +projects_sync_events: :gitlab_main project_statistics: :gitlab_main project_topics: :gitlab_main project_tracing_settings: :gitlab_main diff --git a/spec/lib/gitlab/background_migration/backfill_namespace_hierarchies_spec.rb b/spec/lib/gitlab/background_migration/backfill_namespace_hierarchies_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..e6a6521ac1a5da46c7dc2468b092d8a471a69265 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_namespace_hierarchies_spec.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceHierarchies, :migration, schema: 20211011141240 do + let(:namespaces) { table(:namespaces) } + let(:ci_namespace_hierarchies) { table(:ci_namespace_hierarchies) } + + subject { described_class.new } + + describe '#perform' do + it 'creates hierarchies for all namespaces in range' do + namespaces.create!(id: 5, name: 'test1', path: 'test1') + namespaces.create!(id: 7, name: 'test2', path: 'test2') + namespaces.create!(id: 8, name: 'test3', path: 'test3') + + subject.perform(5, 7, 1) + + expect(ci_namespace_hierarchies.all).to contain_exactly( + an_object_having_attributes(namespace_id: 5, traversal_ids: [5]), + an_object_having_attributes(namespace_id: 7, traversal_ids: [7]) + ) + end + + it 'handles existing hierarchies gracefully' do + namespaces.create!(id: 5, name: 'test1', path: 'test1') + namespaces.create!(id: 7, name: 'test2', path: 'test2') + namespaces.create!(id: 8, name: 'test3', path: 'test3') + + # Simulate a situation where a user has had a chance to move a group to another parent + # before the background migration has had a chance to run + ci_namespace_hierarchies.create!(namespace_id: 7, traversal_ids: [5, 7]) + + subject.perform(5, 7, 1) + + expect(ci_namespace_hierarchies.all).to contain_exactly( + an_object_having_attributes(namespace_id: 5, traversal_ids: [5]), + an_object_having_attributes(namespace_id: 7, traversal_ids: [5, 7]) + ) + end + end +end diff --git a/spec/lib/gitlab/background_migration/backfill_project_hierarchies_spec.rb b/spec/lib/gitlab/background_migration/backfill_project_hierarchies_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..fe475408ba69963a49f866a2d2655e08dd464b03 --- /dev/null +++ b/spec/lib/gitlab/background_migration/backfill_project_hierarchies_spec.rb @@ -0,0 +1,46 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::BackgroundMigration::BackfillProjectHierarchies, :migration, schema: 20211011141241 do + let(:namespaces) { table(:namespaces) } + let(:projects) { table(:projects) } + let(:ci_project_hierarchies) { table(:ci_project_hierarchies) } + + subject { described_class.new } + + describe '#perform' do + it 'creates hierarchies for all projects in range' do + namespaces.create!(id: 10, name: 'namespace1', path: 'namespace1') + projects.create!(id: 5, namespace_id: 10, name: 'test1', path: 'test1') + projects.create!(id: 7, namespace_id: 10, name: 'test2', path: 'test2') + projects.create!(id: 8, namespace_id: 10, name: 'test3', path: 'test3') + + subject.perform(5, 7, 1) + + expect(ci_project_hierarchies.all).to contain_exactly( + an_object_having_attributes(project_id: 5, traversal_ids: [10]), + an_object_having_attributes(project_id: 7, traversal_ids: [10]) + ) + end + + it 'handles existing hierarchies gracefully' do + namespaces.create!(id: 10, name: 'namespace1', path: 'namespace1') + namespaces.create!(id: 11, name: 'namespace2', path: 'namespace2', parent_id: 10) + projects.create!(id: 5, namespace_id: 10, name: 'test1', path: 'test1') + projects.create!(id: 7, namespace_id: 11, name: 'test2', path: 'test2') + projects.create!(id: 8, namespace_id: 11, name: 'test3', path: 'test3') + + # Simulate a situation where a user has had a chance to move a project to another namespace + # before the background migration has had a chance to run + ci_project_hierarchies.create!(project_id: 7, traversal_ids: [10, 11]) + + subject.perform(5, 7, 1) + + expect(ci_project_hierarchies.all).to contain_exactly( + an_object_having_attributes(project_id: 5, traversal_ids: [10]), + an_object_having_attributes(project_id: 7, traversal_ids: [10, 11]) + ) + end + end +end diff --git a/spec/models/ci/namespace_hierarchy_spec.rb b/spec/models/ci/namespace_hierarchy_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..8fc132cb3e0b47942919bdd7d68d6c94944257e5 --- /dev/null +++ b/spec/models/ci/namespace_hierarchy_spec.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::NamespaceHierarchy do + describe '.update_traversal_ids' do + subject { described_class.update_traversal_ids(old_self_and_ancestor_ids, new_self_and_ancestor_ids) } + + context 'when changing parent group' do + before do + described_class.create!( + namespace_id: old_self_and_ancestor_ids.last, + traversal_ids: old_self_and_ancestor_ids + ) + end + + context 'on a top level group' do + let_it_be(:root_group) { create(:group) } + let_it_be(:group) { create(:group) } + + let(:old_self_and_ancestor_ids) { [group.id] } + let(:new_self_and_ancestor_ids) { [root_group, group].map(&:id) } + + it 'updates the related ci_namespace_hierarchy record' do + subject + + expect(described_class.find_by!(namespace_id: group.id)).to have_attributes( + namespace_id: group.id, + traversal_ids: new_self_and_ancestor_ids + ) + end + end + + context 'on a group in the middle of the hierarchy' do + let_it_be(:root_group) { create(:group) } + let_it_be(:sub_group) { create(:group, parent: root_group) } + let_it_be(:group) { create(:group, parent: sub_group) } + + let(:old_self_and_ancestor_ids) { [root_group, sub_group].map(&:id) } + let(:new_self_and_ancestor_ids) { [sub_group].map(&:id) } + + before do + described_class.create!( + namespace_id: group.id, + traversal_ids: [root_group, sub_group, group].map(&:id) + ) + end + + it 'updates the related ci_namespace_hierarchy record' do + subject + + expect(described_class.find(sub_group.id)).to have_attributes( + namespace_id: sub_group.id, + traversal_ids: new_self_and_ancestor_ids + ) + end + + it 'updates the records for children groups with the new hierarchy' do + subject + + expect(described_class.find(group.id)).to have_attributes(traversal_ids: [sub_group, group].map(&:id)) + end + end + end + end +end diff --git a/spec/models/ci/project_hierarchy_spec.rb b/spec/models/ci/project_hierarchy_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..2cfbaff44a35ba62b0ff3b2e6196bad080513463 --- /dev/null +++ b/spec/models/ci/project_hierarchy_spec.rb @@ -0,0 +1,81 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ProjectHierarchy do + let_it_be(:root_group) { create(:group) } + + describe '.update_traversal_ids' do + subject { described_class.update_traversal_ids(project.id, new_self_and_ancestor_ids) } + + before do + described_class.create!(project_id: project.id, traversal_ids: old_self_and_ancestor_ids) + end + + context 'on a top level project' do + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project) } + + let(:old_self_and_ancestor_ids) { [] } + let(:new_self_and_ancestor_ids) { [root_group, group].map(&:id) } + + it 'updates the related ci_project_hierarchy record' do + subject + + expect(described_class.find(project.id)).to have_attributes(traversal_ids: new_self_and_ancestor_ids) + end + end + + context 'on a group in the middle of the hierarchy' do + let_it_be(:sub_group) { create(:group, parent: root_group) } + let_it_be(:group) { create(:group, parent: sub_group) } + let_it_be(:project) { create(:project, group: group) } + + let(:old_self_and_ancestor_ids) { [root_group, sub_group].map(&:id) } + let(:new_self_and_ancestor_ids) { [sub_group].map(&:id) } + + it 'updates the related ci_project_hierarchy record' do + subject + + expect(described_class.find(project.id)).to have_attributes(traversal_ids: new_self_and_ancestor_ids) + end + end + end + + describe '.update_namespace_traversal_ids' do + subject { described_class.update_namespace_traversal_ids(old_self_and_ancestor_ids, new_self_and_ancestor_ids) } + + before do + described_class.create!(project_id: project.id, traversal_ids: old_self_and_ancestor_ids) + end + + context 'on a top level project' do + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, group: group) } + + let(:old_self_and_ancestor_ids) { [group.id] } + let(:new_self_and_ancestor_ids) { [root_group, group].map(&:id) } + + it 'updates the related ci_project_hierarchy record' do + subject + + expect(described_class.find(project.id)).to have_attributes(traversal_ids: new_self_and_ancestor_ids) + end + end + + context 'on a group with 3 levels of hierarchy' do + let_it_be(:sub_group) { create(:group, parent: root_group) } + let_it_be(:group) { create(:group, parent: sub_group) } + let_it_be(:project) { create(:project, group: group) } + + let(:old_self_and_ancestor_ids) { [root_group, sub_group].map(&:id) } + let(:new_self_and_ancestor_ids) { [sub_group].map(&:id) } + + it 'updates the related ci_project_hierarchy record' do + subject + + expect(described_class.find(project.id)).to have_attributes(traversal_ids: new_self_and_ancestor_ids) + end + end + end +end diff --git a/spec/models/ci/runner_spec.rb b/spec/models/ci/runner_spec.rb index 2e79159cc60b852aef620fd4fed7a902f758c19d..7aaec55a9f4c30a7c22cab12316153fd71fa1f0d 100644 --- a/spec/models/ci/runner_spec.rb +++ b/spec/models/ci/runner_spec.rb @@ -547,6 +547,11 @@ def stub_redis_runner_contacted_at(value) let(:group) { create(:group, projects: [build.project]) } let(:runner) { create(:ci_runner, :group, tag_list: tag_list, run_untagged: run_untagged, groups: [group]) } + before do + # Ensure the project namespace is reloaded + build.project.reload + end + it 'can handle builds' do expect(runner.can_pick?(build)).to be_truthy end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 8f5860c799c4df3eb7b2e9049db4872b291e376c..87c707e2ce64d208855049bbc6a901285f91d45c 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -2066,4 +2066,63 @@ def project_rugged(project) it { is_expected.to be(true) } end end + + describe 'namespace syncing' do + subject { child_group } + + let_it_be(:parent_group) { create(:group) } + let_it_be(:parent_group2) { create(:group) } + + let(:child_group) { create(:group, parent: parent_group) } + + it 'creates a matching namespaces_sync_event record for record without parent' do + expect(Namespaces::SyncEvent.where(namespace_id: parent_group.id)).to contain_exactly( + an_object_having_attributes( traversal_ids: [] ) + ) + end + + it 'creates a matching namespaces_sync_event record for record with parent' do + expect(Namespaces::SyncEvent.where(namespace_id: child_group.id)).to contain_exactly( + an_object_having_attributes( traversal_ids: [] ) + ) + end + + context 'when update to new parent is saved' do + let(:sync_events) { Namespaces::SyncEvent.all.to_a.select { |r| r.traversal_ids.include?(subject.id) } } + + it 'creates a matching namespaces_sync_event record' do + subject.update!(parent_id: parent_group2.id) + + expect(sync_events).to contain_exactly( + an_object_having_attributes( + traversal_ids: [parent_group.id, subject.id], + new_traversal_ids: [parent_group2.id, subject.id] + ) + ) + end + end + + context 'when update to new parent is saved twice in same transaction' do + let(:sync_events) { Namespaces::SyncEvent.all.to_a.select { |r| r.traversal_ids.include?(subject.id) } } + + it 'creates two namespaces_sync_event records' do + intermediate_group = create(:group) + Namespace.transaction do + subject.update!(parent_id: intermediate_group.id) + subject.update!(parent_id: parent_group2.id) + end + + expect(sync_events).to match([ + an_object_having_attributes(traversal_ids: [parent_group.id, subject.id], new_traversal_ids: [intermediate_group.id, subject.id]), + an_object_having_attributes(traversal_ids: [intermediate_group.id, subject.id], new_traversal_ids: [parent_group2.id, subject.id]) + ]) + end + end + + specify 'update kicks off ProcessNamespaceSyncEventsWorker' do + expect(Ci::ProcessNamespaceSyncEventsWorker).to receive(:perform_async).twice + + subject.update!(parent_id: parent_group2.id) + end + end end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 8a4e485a298cf8523bb3435d4b3ac631d57b8b3c..4413a1724813eb767e62df9dcb9f1ffce92c9b65 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -7173,7 +7173,7 @@ def has_external_wiki end describe '#enabled_group_deploy_keys' do - let_it_be(:project) { create(:project) } + let(:project) { create(:project) } subject { project.enabled_group_deploy_keys } @@ -7193,7 +7193,6 @@ def has_external_wiki let(:super_group) { create(:group) } it 'returns both group deploy keys' do - super_group = create(:group) super_group_deploy_key = create(:group_deploy_key, groups: [super_group]) group.update!(parent: super_group) @@ -7203,7 +7202,7 @@ def has_external_wiki end context 'and another group has a group deploy key enabled' do - let_it_be(:group_deploy_key) { create(:group_deploy_key) } + let(:group_deploy_key) { create(:group_deploy_key) } it 'does not return this group deploy key' do another_group = create(:group) @@ -7391,6 +7390,56 @@ def has_external_wiki end end + describe 'updating parent namespace' do + subject { project } + + let_it_be(:parent_namespace) { create(:namespace) } + + let(:project) { create(:project) } + let!(:original_namespace) { subject.namespace } + + it 'creates a matching projects_sync_event record' do + expect(Projects::SyncEvent.find_by(project_id: subject.id)).to have_attributes({ + project_id: subject.id, traversal_ids: [], new_traversal_ids: [original_namespace.id] + }) + + subject.update!(namespace_id: parent_namespace.id) + + sync_events = Projects::SyncEvent.where(project_id: subject.id) + + expect(sync_events).to match( + [ + an_object_having_attributes(project_id: subject.id, traversal_ids: [], new_traversal_ids: [original_namespace.id]), + an_object_having_attributes(project_id: subject.id, traversal_ids: [original_namespace.id], new_traversal_ids: [parent_namespace.id]) + ] + ) + end + + context 'twice in the same transaction' do + let(:sync_events) { Projects::SyncEvent.where(project_id: subject.id) } + + it 'creates two update projects_sync_event records' do + intermediate_namespace = create(:namespace) + Project.transaction do + subject.update!(namespace_id: intermediate_namespace.id) + subject.update!(namespace_id: parent_namespace.id) + end + + expect(sync_events).to match([ + an_object_having_attributes(project_id: subject.id, traversal_ids: [], new_traversal_ids: [original_namespace.id]), + an_object_having_attributes(project_id: subject.id, traversal_ids: [original_namespace.id], new_traversal_ids: [intermediate_namespace.id]), + an_object_having_attributes(project_id: subject.id, traversal_ids: [intermediate_namespace.id], new_traversal_ids: [parent_namespace.id]) + ]) + end + end + + it 'kicks off ProcessProjectSyncEventsWorker' do + expect(Ci::ProcessProjectSyncEventsWorker).to receive(:perform_async) + + subject.update!(namespace_id: parent_namespace.id) + end + end + def finish_job(export_job) export_job.start export_job.finish diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index b5d4614d206063e6e5772875535c4cf0f02dafe6..b01c71a95682f1b9dac051976add31ae5dc1cd63 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -1540,7 +1540,8 @@ allow(user).to receive(:update_highest_role) end - expect(SecureRandom).to receive(:hex).and_return('3b8ca303') + allow(SecureRandom).to receive(:hex).and_call_original + expect(SecureRandom).to receive(:hex).with(no_args).and_return('3b8ca303') user = create(:user) diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index 097d374640c66b9887dadfc814ea26998b864d32..3ed08afd57db4d201dab5e0ff792e4bbc82dcdd8 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -47,7 +47,7 @@ it 'executes a limited number of queries' do control_count = ActiveRecord::QueryRecorder.new { subject }.count - expect(control_count).to be <= 101 + expect(control_count).to be <= 104 end it 'schedules an import using a namespace' do diff --git a/spec/services/notification_service_spec.rb b/spec/services/notification_service_spec.rb index fbf5b1833658ad892ecf2e0c5826c3a391b13f94..e8f14c723aa05dffbf5f73049563716021636bec 100644 --- a/spec/services/notification_service_spec.rb +++ b/spec/services/notification_service_spec.rb @@ -2942,7 +2942,7 @@ def create_member! describe 'Pipelines', :deliver_mails_inline do describe '#pipeline_finished' do - let_it_be(:project) { create(:project, :public, :repository) } + let_it_be_with_reload(:project) { create(:project, :public, :repository) } let_it_be(:u_member) { create(:user) } let_it_be(:u_watcher) { create_user_with_notification(:watch, 'watcher') } @@ -3088,6 +3088,7 @@ def create_pipeline(user, status) let(:group_notification_email) { 'user+group@example.com' } before do + project.reload group = create(:group) project.update!(group: group) @@ -3551,6 +3552,7 @@ def build_team(project) # with different notification settings def build_group(project, visibility: :public) group = create_nested_group(visibility) + project.reload project.update!(namespace_id: group.id) # Group member: global=disabled, group=watch diff --git a/spec/workers/ci/process_namespace_sync_events_worker_spec.rb b/spec/workers/ci/process_namespace_sync_events_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..3490f1b84f326b3ec8b1350f1530b7ee53c92716 --- /dev/null +++ b/spec/workers/ci/process_namespace_sync_events_worker_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ProcessNamespaceSyncEventsWorker do + describe '#perform' do + subject(:perform) { described_class.new.perform } + + context 'on namespace with group changing parent multiple times' do + let_it_be(:group) { create(:group) } + let_it_be(:parent_group_1) { create(:group) } + let_it_be(:parent_group_2) { create(:group) } + + let(:expected_initial_sync_event_count) { 3 } + let(:new_sync_events) do + [ + { namespace_id: group.id, traversal_ids: [group.id], new_traversal_ids: [parent_group_1.id, group.id] }, + { namespace_id: group.id, traversal_ids: [parent_group_1.id, group.id], new_traversal_ids: [parent_group_2.id, group.id] }, + { namespace_id: parent_group_2.id, traversal_ids: [parent_group_2.id], new_traversal_ids: [parent_group_1.id, parent_group_2.id] } + ] + end + + before do + # Group creation from let_it_be causes 3 records to be implicitly created + expect(Namespaces::SyncEvent.count).to eq(expected_initial_sync_event_count) + + Namespaces::SyncEvent.insert_all! new_sync_events + end + + include_examples 'an idempotent worker' do + it 'consumes all sync events' do + expect { subject }.to change(Namespaces::SyncEvent, :count).from(expected_initial_sync_event_count + new_sync_events.count).to(0) + end + + it 'syncs namespace hierarchy traversal ids' do + expect { subject }.to change(Ci::NamespaceHierarchy, :all).to contain_exactly( + an_object_having_attributes(namespace_id: parent_group_1.id, traversal_ids: [parent_group_1.id]), + an_object_having_attributes(namespace_id: parent_group_2.id, traversal_ids: [parent_group_1.id, parent_group_2.id]), + an_object_having_attributes(namespace_id: group.id, traversal_ids: [parent_group_1.id, parent_group_2.id, group.id]) + ) + end + end + end + end +end diff --git a/spec/workers/ci/process_project_sync_events_worker_spec.rb b/spec/workers/ci/process_project_sync_events_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..56ee23828ea4bdb760245c22295a0ab63ee7e70f --- /dev/null +++ b/spec/workers/ci/process_project_sync_events_worker_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::ProcessProjectSyncEventsWorker do + describe '#perform' do + subject(:perform) { described_class.new.perform } + + include_examples 'an idempotent worker' do + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, group: group) } + let_it_be(:parent_group_1) { create(:group) } + + let(:expected_initial_sync_event_count) { 1 } + let(:new_sync_records) do + [{ project_id: project.id, traversal_ids: [group.id], new_traversal_ids: [parent_group_1.id, group.id] }] + end + + before do + # Project creation from let_it_be causes one record to be implicitly created + expect(Projects::SyncEvent.where(project_id: project.id).count).to eq(expected_initial_sync_event_count) + + Projects::SyncEvent.insert_all! new_sync_records + end + + it 'consumes all sync events' do + expect { subject }.to change(Projects::SyncEvent, :count).from(expected_initial_sync_event_count + new_sync_records.count).to(0) + end + + it 'syncs project hierarchy traversal ids' do + expect { subject }.to change(Ci::ProjectHierarchy, :all).to contain_exactly( + an_object_having_attributes(project_id: project.id, traversal_ids: [parent_group_1.id, group.id]) + ) + end + end + + context 'on project changing parent multiple times' do + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, group: group) } + let_it_be(:parent_group_1) { create(:group) } + let_it_be(:parent_group_2) { create(:group) } + + before do + Projects::SyncEvent.insert_all! [ + { project_id: project.id, traversal_ids: [group.id], new_traversal_ids: [parent_group_1.id, group.id] }, + { project_id: project.id, traversal_ids: [parent_group_1.id, group.id], new_traversal_ids: [parent_group_2.id, group.id] }, + { project_id: project.id, traversal_ids: [parent_group_2.id, group.id], new_traversal_ids: [parent_group_1.id, parent_group_2.id, group.id] } + ] + end + + it 'consumes all sync events' do + expect { subject }.to change(Projects::SyncEvent, :count).to(0) + end + + it 'syncs project hierarchy traversal ids' do + expect { subject }.to change(Ci::ProjectHierarchy.where(project_id: project.id), :all).to contain_exactly( + an_object_having_attributes(traversal_ids: [parent_group_1.id, parent_group_2.id, group.id]) + ) + end + end + end +end