From f35e25cfa06ec198841ddba92c36ed8df4188755 Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Wed, 19 Feb 2025 11:35:19 +0530 Subject: [PATCH 01/12] Rename remote development namespace cluster agent mapping table Changelog: other EE: true --- db/docs/namespace_cluster_agent_mappings.yml | 14 ++++ ...pment_namespace_cluster_agent_mappings.yml | 9 +++ ...lopment_namespace_cluster_agent_mapping.rb | 16 +++++ db/schema_migrations/20250219055827 | 1 + db/structure.sql | 65 +++++++++++-------- 5 files changed, 77 insertions(+), 28 deletions(-) create mode 100644 db/docs/namespace_cluster_agent_mappings.yml create mode 100644 db/docs/views/remote_development_namespace_cluster_agent_mappings.yml create mode 100644 db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb create mode 100644 db/schema_migrations/20250219055827 diff --git a/db/docs/namespace_cluster_agent_mappings.yml b/db/docs/namespace_cluster_agent_mappings.yml new file mode 100644 index 00000000000000..fc3e03e0ab9388 --- /dev/null +++ b/db/docs/namespace_cluster_agent_mappings.yml @@ -0,0 +1,14 @@ +--- +table_name: namespace_cluster_agent_mappings +classes: +- RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping +feature_categories: +- workspaces +description: This table records associations between Namespaces and Cluster Agents + as a part of supporting group-cluster agent association in the context of Workspaces +introduced_by_url: '' +milestone: '17.10' +gitlab_schema: gitlab_main_cell +sharding_key: + namespace_id: namespaces +table_size: small diff --git a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml new file mode 100644 index 00000000000000..e0121f22ffba83 --- /dev/null +++ b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml @@ -0,0 +1,9 @@ +--- +view_name: remote_development_namespace_cluster_agent_mappings +classes: +- RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping +feature_categories: [] +description: +introduced_by_url: +milestone: '17.9' +table_size: small diff --git a/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb b/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb new file mode 100644 index 00000000000000..cebb288f8401fb --- /dev/null +++ b/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class RenameRemoteDevelopmentNamespaceClusterAgentMapping < Gitlab::Database::Migration[2.2] + milestone '17.9' + + def up + rename_table_safely(:remote_development_namespace_cluster_agent_mappings, :namespace_cluster_agent_mappings) + end + + def down + undo_rename_table_safely(:remote_development_namespace_cluster_agent_mappings, :namespace_cluster_agent_mappings) + end +end diff --git a/db/schema_migrations/20250219055827 b/db/schema_migrations/20250219055827 new file mode 100644 index 00000000000000..388e56e8551869 --- /dev/null +++ b/db/schema_migrations/20250219055827 @@ -0,0 +1 @@ +166459c440ed6e868c66ddc3b79a5aae27b20d2bc21e489123b6676f8cd97841 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 56db66feda0a28..ee326ee27955c7 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -16895,6 +16895,24 @@ CREATE TABLE namespace_ci_cd_settings ( allow_stale_runner_pruning boolean DEFAULT false NOT NULL ); +CREATE TABLE namespace_cluster_agent_mappings ( + id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + namespace_id bigint NOT NULL, + cluster_agent_id bigint NOT NULL, + creator_id bigint +); + +CREATE SEQUENCE namespace_cluster_agent_mappings_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE namespace_cluster_agent_mappings_id_seq OWNED BY namespace_cluster_agent_mappings.id; + CREATE TABLE namespace_commit_emails ( id bigint NOT NULL, user_id bigint NOT NULL, @@ -20725,23 +20743,14 @@ CREATE SEQUENCE releases_id_seq ALTER SEQUENCE releases_id_seq OWNED BY releases.id; -CREATE TABLE remote_development_namespace_cluster_agent_mappings ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - namespace_id bigint NOT NULL, - cluster_agent_id bigint NOT NULL, - creator_id bigint -); - -CREATE SEQUENCE remote_development_namespace_cluster_agent_mappings_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE remote_development_namespace_cluster_agent_mappings_id_seq OWNED BY remote_development_namespace_cluster_agent_mappings.id; +CREATE VIEW remote_development_namespace_cluster_agent_mappings AS + SELECT namespace_cluster_agent_mappings.id, + namespace_cluster_agent_mappings.created_at, + namespace_cluster_agent_mappings.updated_at, + namespace_cluster_agent_mappings.namespace_id, + namespace_cluster_agent_mappings.cluster_agent_id, + namespace_cluster_agent_mappings.creator_id + FROM namespace_cluster_agent_mappings; CREATE TABLE remote_mirrors ( id bigint NOT NULL, @@ -25868,6 +25877,8 @@ ALTER TABLE ONLY namespace_admin_notes ALTER COLUMN id SET DEFAULT nextval('name ALTER TABLE ONLY namespace_bans ALTER COLUMN id SET DEFAULT nextval('namespace_bans_id_seq'::regclass); +ALTER TABLE ONLY namespace_cluster_agent_mappings ALTER COLUMN id SET DEFAULT nextval('namespace_cluster_agent_mappings_id_seq'::regclass); + ALTER TABLE ONLY namespace_commit_emails ALTER COLUMN id SET DEFAULT nextval('namespace_commit_emails_id_seq'::regclass); ALTER TABLE ONLY namespace_import_users ALTER COLUMN id SET DEFAULT nextval('namespace_import_users_id_seq'::regclass); @@ -26144,8 +26155,6 @@ ALTER TABLE ONLY release_links ALTER COLUMN id SET DEFAULT nextval('release_link ALTER TABLE ONLY releases ALTER COLUMN id SET DEFAULT nextval('releases_id_seq'::regclass); -ALTER TABLE ONLY remote_development_namespace_cluster_agent_mappings ALTER COLUMN id SET DEFAULT nextval('remote_development_namespace_cluster_agent_mappings_id_seq'::regclass); - ALTER TABLE ONLY remote_mirrors ALTER COLUMN id SET DEFAULT nextval('remote_mirrors_id_seq'::regclass); ALTER TABLE ONLY required_code_owners_sections ALTER COLUMN id SET DEFAULT nextval('required_code_owners_sections_id_seq'::regclass); @@ -28447,6 +28456,9 @@ ALTER TABLE ONLY namespace_bans ALTER TABLE ONLY namespace_ci_cd_settings ADD CONSTRAINT namespace_ci_cd_settings_pkey PRIMARY KEY (namespace_id); +ALTER TABLE ONLY namespace_cluster_agent_mappings + ADD CONSTRAINT namespace_cluster_agent_mappings_pkey PRIMARY KEY (id); + ALTER TABLE ONLY namespace_commit_emails ADD CONSTRAINT namespace_commit_emails_pkey PRIMARY KEY (id); @@ -28996,9 +29008,6 @@ ALTER TABLE releases ALTER TABLE ONLY releases ADD CONSTRAINT releases_pkey PRIMARY KEY (id); -ALTER TABLE ONLY remote_development_namespace_cluster_agent_mappings - ADD CONSTRAINT remote_development_namespace_cluster_agent_mappings_pkey PRIMARY KEY (id); - ALTER TABLE ONLY remote_mirrors ADD CONSTRAINT remote_mirrors_pkey PRIMARY KEY (id); @@ -31072,9 +31081,9 @@ CREATE UNIQUE INDEX i_duo_workflows_events_on_correlation_id ON duo_workflows_ev CREATE INDEX i_gitlab_subscription_histories_on_namespace_change_type_plan ON gitlab_subscription_histories USING btree (namespace_id, change_type, hosted_plan_id); -CREATE INDEX i_namespace_cluster_agent_mappings_on_cluster_agent_id ON remote_development_namespace_cluster_agent_mappings USING btree (cluster_agent_id); +CREATE INDEX i_namespace_cluster_agent_mappings_on_cluster_agent_id ON namespace_cluster_agent_mappings USING btree (cluster_agent_id); -CREATE INDEX i_namespace_cluster_agent_mappings_on_creator_id ON remote_development_namespace_cluster_agent_mappings USING btree (creator_id); +CREATE INDEX i_namespace_cluster_agent_mappings_on_creator_id ON namespace_cluster_agent_mappings USING btree (creator_id); CREATE INDEX i_organization_cluster_agent_mappings_on_creator_id ON organization_cluster_agent_mappings USING btree (creator_id); @@ -36412,7 +36421,7 @@ CREATE UNIQUE INDEX unique_merge_request_metrics_by_merge_request_id ON merge_re CREATE INDEX unique_ml_model_versions_on_model_id_and_id ON ml_model_versions USING btree (model_id, id DESC); -CREATE UNIQUE INDEX unique_namespace_cluster_agent_mappings_for_agent_association ON remote_development_namespace_cluster_agent_mappings USING btree (namespace_id, cluster_agent_id); +CREATE UNIQUE INDEX unique_namespace_cluster_agent_mappings_for_agent_association ON namespace_cluster_agent_mappings USING btree (namespace_id, cluster_agent_id); CREATE UNIQUE INDEX unique_organizations_on_path_case_insensitive ON organizations USING btree (lower(path)); @@ -38985,7 +38994,7 @@ ALTER TABLE ONLY audit_events_amazon_s3_configurations ALTER TABLE ONLY issue_customer_relations_contacts ADD CONSTRAINT fk_0c0037f723 FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE; -ALTER TABLE ONLY remote_development_namespace_cluster_agent_mappings +ALTER TABLE ONLY namespace_cluster_agent_mappings ADD CONSTRAINT fk_0c483ecb9d FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; ALTER TABLE ONLY zoekt_replicas @@ -39036,7 +39045,7 @@ ALTER TABLE ONLY protected_environment_deploy_access_levels ALTER TABLE ONLY cluster_agent_migrations ADD CONSTRAINT fk_1211a345fb FOREIGN KEY (agent_id) REFERENCES cluster_agents(id) ON DELETE CASCADE; -ALTER TABLE ONLY remote_development_namespace_cluster_agent_mappings +ALTER TABLE ONLY namespace_cluster_agent_mappings ADD CONSTRAINT fk_124d8167c5 FOREIGN KEY (creator_id) REFERENCES users(id) ON DELETE SET NULL; ALTER TABLE ONLY cluster_agent_url_configurations @@ -40401,7 +40410,7 @@ ALTER TABLE ONLY ci_sources_pipelines ALTER TABLE ONLY packages_maven_metadata ADD CONSTRAINT fk_be88aed360 FOREIGN KEY (package_id) REFERENCES packages_packages(id) ON DELETE CASCADE; -ALTER TABLE ONLY remote_development_namespace_cluster_agent_mappings +ALTER TABLE ONLY namespace_cluster_agent_mappings ADD CONSTRAINT fk_be8e9c740f FOREIGN KEY (cluster_agent_id) REFERENCES cluster_agents(id) ON DELETE CASCADE; ALTER TABLE ONLY oauth_device_grants -- GitLab From 450a760ba01cbd30e1a3b5c46a2d022e7cd7a430 Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Wed, 19 Feb 2025 12:45:08 +0530 Subject: [PATCH 02/12] Rename the namespace cluste mapping table remote_development_namespace_cluster_agent_mappings to namespace_cluster_agent_mappings More info: Spin off based on the following comment: https://gitlab.com/gitlab-org/gitlab/-/issues/513366#note_2317269171 As part of introducing the organization_cluster_agent_mappings table, we plan on renaming remote_development_namespace_cluster_agent_mappings to namespace_cluster_agent_mappings as the prefix adds no value and makes things long. --- db/docs/namespace_cluster_agent_mappings.yml | 2 +- ...mote_development_namespace_cluster_agent_mappings.yml | 2 +- ...mote_development_namespace_cluster_agent_mappings.yml | 7 ++++--- .../finders/remote_development/cluster_agents_finder.rb | 6 +++--- .../remote_development/workspace_operations/create.rb | 2 +- ee/app/models/ee/clusters/agent.rb | 4 ++-- ee/app/models/ee/namespace.rb | 4 ++-- ee/app/models/ee/user.rb | 4 ++-- ...ent_mapping.rb => namespace_cluster_agent_mapping.rb} | 9 +++++---- .../backfill_root_namespace_cluster_agent_mappings.rb | 8 ++++---- .../create/mapping_creator.rb | 2 +- .../delete/mapping_deleter.rb | 2 +- .../validations.rb | 4 ++-- ...emote_development_namespace_cluster_agent_mappings.rb | 2 +- ...ackfill_root_namespace_cluster_agent_mappings_spec.rb | 2 +- .../create/main_integration_spec.rb | 6 +++--- .../create/mapping_creator_spec.rb | 2 +- .../delete/main_integration_spec.rb | 4 ++-- .../delete/mapping_deleter_spec.rb | 2 +- ...g_spec.rb => namespace_cluster_agent_mapping_spec.rb} | 8 ++++---- .../create_spec.rb | 2 +- 21 files changed, 43 insertions(+), 41 deletions(-) rename ee/app/models/remote_development/{remote_development_namespace_cluster_agent_mapping.rb => namespace_cluster_agent_mapping.rb} (62%) rename ee/spec/models/remote_development/{remote_development_namespace_cluster_agent_mapping_spec.rb => namespace_cluster_agent_mapping_spec.rb} (79%) diff --git a/db/docs/namespace_cluster_agent_mappings.yml b/db/docs/namespace_cluster_agent_mappings.yml index fc3e03e0ab9388..2e9bb929ac7732 100644 --- a/db/docs/namespace_cluster_agent_mappings.yml +++ b/db/docs/namespace_cluster_agent_mappings.yml @@ -1,7 +1,7 @@ --- table_name: namespace_cluster_agent_mappings classes: -- RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping +- RemoteDevelopment::NamespaceClusterAgentMapping feature_categories: - workspaces description: This table records associations between Namespaces and Cluster Agents diff --git a/db/docs/remote_development_namespace_cluster_agent_mappings.yml b/db/docs/remote_development_namespace_cluster_agent_mappings.yml index 21a94c0092bcb5..1fdd3339692ff2 100644 --- a/db/docs/remote_development_namespace_cluster_agent_mappings.yml +++ b/db/docs/remote_development_namespace_cluster_agent_mappings.yml @@ -1,7 +1,7 @@ --- table_name: remote_development_namespace_cluster_agent_mappings classes: -- RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping +- RemoteDevelopment::NamespaceClusterAgentMapping feature_categories: - workspaces description: This table records associations between Namespaces and Cluster Agents diff --git a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml index e0121f22ffba83..b5356054343752 100644 --- a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml +++ b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml @@ -1,9 +1,10 @@ --- view_name: remote_development_namespace_cluster_agent_mappings classes: -- RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping +- RemoteDevelopment::NamespaceClusterAgentMapping feature_categories: [] description: -introduced_by_url: -milestone: '17.9' +introduced_by_url: '' +gitlab_schema: gitlab_main_cell +milestone: '17.10' table_size: small diff --git a/ee/app/finders/remote_development/cluster_agents_finder.rb b/ee/app/finders/remote_development/cluster_agents_finder.rb index 892ad043265d3d..68bf128175f16f 100644 --- a/ee/app/finders/remote_development/cluster_agents_finder.rb +++ b/ee/app/finders/remote_development/cluster_agents_finder.rb @@ -15,7 +15,7 @@ def self.fetch_agents(filter, namespace, user) # noinspection RailsParamDefResolve -- A symbol is a valid argument for 'select' existing_mapped_agents = - RemoteDevelopmentNamespaceClusterAgentMapping + NamespaceClusterAgentMapping .for_namespaces([namespace.id]) .select(:cluster_agent_id) @@ -28,7 +28,7 @@ def self.fetch_agents(filter, namespace, user) when :directly_mapped return Clusters::Agent.none unless user_can_read_namespace_agent_mappings?(user: user, namespace: namespace) - relevant_mappings = RemoteDevelopmentNamespaceClusterAgentMapping.for_namespaces([namespace.id]) + relevant_mappings = NamespaceClusterAgentMapping.for_namespaces([namespace.id]) relevant_mappings = NamespaceClusterAgentMappingOperations::Validations.filter_valid_namespace_cluster_agent_mappings( namespace_cluster_agent_mappings: relevant_mappings.to_a @@ -36,7 +36,7 @@ def self.fetch_agents(filter, namespace, user) Clusters::Agent.id_in(relevant_mappings.map(&:cluster_agent_id)) when :available - relevant_mappings = RemoteDevelopmentNamespaceClusterAgentMapping.for_namespaces(namespace.traversal_ids) + relevant_mappings = NamespaceClusterAgentMapping.for_namespaces(namespace.traversal_ids) relevant_mappings = NamespaceClusterAgentMappingOperations::Validations.filter_valid_namespace_cluster_agent_mappings( namespace_cluster_agent_mappings: relevant_mappings.to_a diff --git a/ee/app/graphql/mutations/remote_development/workspace_operations/create.rb b/ee/app/graphql/mutations/remote_development/workspace_operations/create.rb index d9b92d0f0a2df4..366f7c9760c375 100644 --- a/ee/app/graphql/mutations/remote_development/workspace_operations/create.rb +++ b/ee/app/graphql/mutations/remote_development/workspace_operations/create.rb @@ -100,7 +100,7 @@ def resolve(args) # noinspection RubyNilAnalysis - RubyMine thinks project or agent may be nil, but this is not possible # because authorized_find! would have thrown an exception. relevant_mappings = - ::RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping + ::RemoteDevelopment::NamespaceClusterAgentMapping .for_namespaces(project.project_namespace.traversal_ids) .for_agents([agent.id]) diff --git a/ee/app/models/ee/clusters/agent.rb b/ee/app/models/ee/clusters/agent.rb index 9ca0028e314170..3cf72dfeed2324 100644 --- a/ee/app/models/ee/clusters/agent.rb +++ b/ee/app/models/ee/clusters/agent.rb @@ -36,8 +36,8 @@ module Agent inverse_of: :agent, foreign_key: :cluster_agent_id - has_many :remote_development_namespace_cluster_agent_mappings, - class_name: 'RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping', + has_many :namespace_cluster_agent_mappings, + class_name: 'RemoteDevelopment::NamespaceClusterAgentMapping', inverse_of: :agent, foreign_key: 'cluster_agent_id' diff --git a/ee/app/models/ee/namespace.rb b/ee/app/models/ee/namespace.rb index 2cceb9e85f4e25..fee2874a8047ea 100644 --- a/ee/app/models/ee/namespace.rb +++ b/ee/app/models/ee/namespace.rb @@ -66,8 +66,8 @@ module Namespace has_one :zoekt_enabled_namespace, class_name: '::Search::Zoekt::EnabledNamespace', foreign_key: :root_namespace_id, inverse_of: :namespace - has_many :remote_development_namespace_cluster_agent_mappings, - class_name: 'RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping', + has_many :namespace_cluster_agent_mappings, + class_name: 'RemoteDevelopment::NamespaceClusterAgentMapping', foreign_key: 'namespace_id', inverse_of: :namespace has_many :hosted_runner_monthly_usages, diff --git a/ee/app/models/ee/user.rb b/ee/app/models/ee/user.rb index 4fe9f54c035dec..3642dd4a495e5d 100644 --- a/ee/app/models/ee/user.rb +++ b/ee/app/models/ee/user.rb @@ -133,8 +133,8 @@ module User has_many :assigned_add_ons, class_name: 'GitlabSubscriptions::UserAddOnAssignment', inverse_of: :user, dependent: :destroy # rubocop:enable Cop/ActiveRecordDependent -- legacy usage - has_many :created_remote_development_namespace_cluster_agent_mappings, - class_name: 'RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping', + has_many :created_namespace_cluster_agent_mappings, + class_name: 'RemoteDevelopment::NamespaceClusterAgentMapping', inverse_of: :user has_many :created_organization_cluster_agent_mappings, diff --git a/ee/app/models/remote_development/remote_development_namespace_cluster_agent_mapping.rb b/ee/app/models/remote_development/namespace_cluster_agent_mapping.rb similarity index 62% rename from ee/app/models/remote_development/remote_development_namespace_cluster_agent_mapping.rb rename to ee/app/models/remote_development/namespace_cluster_agent_mapping.rb index 6aab0943549754..1e0df8140c7ead 100644 --- a/ee/app/models/remote_development/remote_development_namespace_cluster_agent_mapping.rb +++ b/ee/app/models/remote_development/namespace_cluster_agent_mapping.rb @@ -1,16 +1,17 @@ # frozen_string_literal: true module RemoteDevelopment - class RemoteDevelopmentNamespaceClusterAgentMapping < ApplicationRecord - belongs_to :namespace, inverse_of: :remote_development_namespace_cluster_agent_mappings + # model + class NamespaceClusterAgentMapping < ApplicationRecord + belongs_to :namespace, inverse_of: :namespace_cluster_agent_mappings belongs_to :user, class_name: 'User', foreign_key: 'creator_id', - inverse_of: :created_remote_development_namespace_cluster_agent_mappings + inverse_of: :created_namespace_cluster_agent_mappings belongs_to :agent, class_name: 'Clusters::Agent', foreign_key: 'cluster_agent_id', - inverse_of: :remote_development_namespace_cluster_agent_mappings + inverse_of: :namespace_cluster_agent_mappings validates :namespace, presence: true validates :agent, presence: true diff --git a/ee/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb b/ee/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb index a5aa644e3d8ccf..2496d0026a4c6f 100644 --- a/ee/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb +++ b/ee/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb @@ -38,13 +38,13 @@ class ClusterAgent < ::ApplicationRecord self.table_name = 'cluster_agents' end - class RemoteDevelopmentNamespaceClusterAgentMapping < ::ApplicationRecord + class NamespaceClusterAgentMapping < ::ApplicationRecord extend LogUtils - self.table_name = 'remote_development_namespace_cluster_agent_mappings' + self.table_name = 'namespace_cluster_agent_mappings' INSERT_SQL = <<~SQL.squish - INSERT INTO remote_development_namespace_cluster_agent_mappings + INSERT INTO namespace_cluster_agent_mappings (namespace_id, cluster_agent_id, creator_id, created_at, updated_at) VALUES %{insert_tuples} @@ -99,7 +99,7 @@ def perform } end - RemoteDevelopmentNamespaceClusterAgentMapping.insert_namespace_cluster_agent_mappings(mappings) + NamespaceClusterAgentMapping.insert_namespace_cluster_agent_mappings(mappings) log_info(message: 'Migration ended for batch', start_id: ids.min, end_id: ids.max) end end diff --git a/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/create/mapping_creator.rb b/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/create/mapping_creator.rb index dd0b9f8b248020..aa4cac8790d8ce 100644 --- a/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/create/mapping_creator.rb +++ b/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/create/mapping_creator.rb @@ -15,7 +15,7 @@ def self.create(context) user: User => user } - new_mapping = RemoteDevelopmentNamespaceClusterAgentMapping.new( + new_mapping = NamespaceClusterAgentMapping.new( cluster_agent_id: cluster_agent.id, namespace_id: namespace.id, creator_id: user.id diff --git a/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/mapping_deleter.rb b/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/mapping_deleter.rb index 1055a46c0f94e1..cd82387d0d47e4 100644 --- a/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/mapping_deleter.rb +++ b/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/mapping_deleter.rb @@ -14,7 +14,7 @@ def self.delete(context) cluster_agent: Clusters::Agent => cluster_agent } - delete_count = RemoteDevelopmentNamespaceClusterAgentMapping.delete_by( + delete_count = NamespaceClusterAgentMapping.delete_by( namespace_id: namespace.id, cluster_agent_id: cluster_agent.id ) diff --git a/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/validations.rb b/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/validations.rb index 96fd63f4c99d69..27509ad3de318c 100644 --- a/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/validations.rb +++ b/ee/lib/remote_development/namespace_cluster_agent_mapping_operations/validations.rb @@ -7,8 +7,8 @@ class Validations # agents are excluded from the response. A cluster agent is said to reside within a namespace # if the namespace id is present in the traversal ids of the project bound to the cluster agent # - # @param [Array] namespace_cluster_agent_mappings - # @return [Array] + # @param [Array] namespace_cluster_agent_mappings + # @return [Array] def self.filter_valid_namespace_cluster_agent_mappings(namespace_cluster_agent_mappings:) agent_ids = namespace_cluster_agent_mappings.map(&:cluster_agent_id) traversal_ids_for_agents = traversal_ids_for_cluster_agents(cluster_agent_ids: agent_ids) diff --git a/ee/spec/factories/remote_development/remote_development_namespace_cluster_agent_mappings.rb b/ee/spec/factories/remote_development/remote_development_namespace_cluster_agent_mappings.rb index 5bd5853dff2f5e..74f74220da7836 100644 --- a/ee/spec/factories/remote_development/remote_development_namespace_cluster_agent_mappings.rb +++ b/ee/spec/factories/remote_development/remote_development_namespace_cluster_agent_mappings.rb @@ -2,7 +2,7 @@ FactoryBot.define do factory :remote_development_namespace_cluster_agent_mapping, - class: 'RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping' do + class: 'RemoteDevelopment::NamespaceClusterAgentMapping' do user agent factory: [:cluster_agent, :in_group] namespace { agent.project.namespace } diff --git a/ee/spec/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings_spec.rb b/ee/spec/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings_spec.rb index d06229feec1fd9..695602d1388d00 100644 --- a/ee/spec/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings_spec.rb +++ b/ee/spec/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings_spec.rb @@ -9,7 +9,7 @@ let(:projects) { table(:projects) } let(:cluster_agents) { table(:cluster_agents) } let(:rd_agent_configs) { table(:remote_development_agent_configs) } - let(:rd_namespace_cluster_agent_mappings) { table(:remote_development_namespace_cluster_agent_mappings) } + let(:rd_namespace_cluster_agent_mappings) { table(:namespace_cluster_agent_mappings) } let!(:organization) { organizations.create!(name: 'organization', path: 'organization') } diff --git a/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/create/main_integration_spec.rb b/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/create/main_integration_spec.rb index e96d5e8e969a99..8c2e86fd67bbe8 100644 --- a/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/create/main_integration_spec.rb +++ b/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/create/main_integration_spec.rb @@ -19,7 +19,7 @@ context 'when params are valid' do it 'creates a new mapping for the given namespace and cluster agent and returns success' do - expect { response }.to change { RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping.count }.by(1) + expect { response }.to change { RemoteDevelopment::NamespaceClusterAgentMapping.count }.by(1) expect(response.fetch(:status)).to eq(:success) expect(response[:message]).to be_nil @@ -44,7 +44,7 @@ let(:cluster_agent) { create(:cluster_agent) } it 'does not create the mapping and returns an error' do - expect { response }.not_to change { RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping.count } + expect { response }.not_to change { RemoteDevelopment::NamespaceClusterAgentMapping.count } expect(response).to eq({ status: :error, @@ -61,7 +61,7 @@ end it 'does not create the mapping and returns an error' do - expect { response }.not_to change { RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping.count } + expect { response }.not_to change { RemoteDevelopment::NamespaceClusterAgentMapping.count } expect(response).to eq({ status: :error, diff --git a/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/create/mapping_creator_spec.rb b/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/create/mapping_creator_spec.rb index 5e0ac3897549e1..4ef376dc7eb152 100644 --- a/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/create/mapping_creator_spec.rb +++ b/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/create/mapping_creator_spec.rb @@ -28,7 +28,7 @@ context 'when the mapping creation fails' do shared_examples 'err result' do |expected_error_details:| it 'does not create the db records and returns an error result containing a failed message with model errors' do - expect { result }.not_to change { RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping.count } + expect { result }.not_to change { RemoteDevelopment::NamespaceClusterAgentMapping.count } expect(result).to be_err_result do |message| expect(message).to be_a(RemoteDevelopment::Messages::NamespaceClusterAgentMappingCreateFailed) diff --git a/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/main_integration_spec.rb b/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/main_integration_spec.rb index c4f41fb77f0a7b..73f94f23cf97ad 100644 --- a/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/main_integration_spec.rb +++ b/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/main_integration_spec.rb @@ -17,7 +17,7 @@ context 'when params are valid' do it 'deletes an existing mapping for a given namespace and cluster_agent' do - expect { response }.to change { RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping.count }.by(-1) + expect { response }.to change { RemoteDevelopment::NamespaceClusterAgentMapping.count }.by(-1) expect(response.fetch(:status)).to eq(:success) expect(response[:message]).to be_nil @@ -32,7 +32,7 @@ end it 'does not create the mapping and returns an error' do - expect { response }.not_to change { RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping.count } + expect { response }.not_to change { RemoteDevelopment::NamespaceClusterAgentMapping.count } expect(response).to eq({ status: :error, diff --git a/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/mapping_deleter_spec.rb b/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/mapping_deleter_spec.rb index ea444fe83f1798..f8e7a4b1134d23 100644 --- a/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/mapping_deleter_spec.rb +++ b/ee/spec/lib/remote_development/namespace_cluster_agent_mapping_operations/delete/mapping_deleter_spec.rb @@ -24,7 +24,7 @@ let(:creator) { create(:user) } before do - RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping.new( + RemoteDevelopment::NamespaceClusterAgentMapping.new( namespace_id: namespace.id, cluster_agent_id: agent.id, creator_id: creator.id diff --git a/ee/spec/models/remote_development/remote_development_namespace_cluster_agent_mapping_spec.rb b/ee/spec/models/remote_development/namespace_cluster_agent_mapping_spec.rb similarity index 79% rename from ee/spec/models/remote_development/remote_development_namespace_cluster_agent_mapping_spec.rb rename to ee/spec/models/remote_development/namespace_cluster_agent_mapping_spec.rb index 86731d03a270b8..e043715acadfb0 100644 --- a/ee/spec/models/remote_development/remote_development_namespace_cluster_agent_mapping_spec.rb +++ b/ee/spec/models/remote_development/namespace_cluster_agent_mapping_spec.rb @@ -3,7 +3,7 @@ require 'spec_helper' # noinspection RubyArgCount -- Rubymine detecting wrong types, it thinks some #create are from Minitest, not FactoryBot -RSpec.describe RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping, feature_category: :workspaces do +RSpec.describe RemoteDevelopment::NamespaceClusterAgentMapping, feature_category: :workspaces do let_it_be(:user) { create(:user) } let_it_be_with_reload(:namespace) { create(:group) } let_it_be_with_reload(:agent) { create(:cluster_agent) } @@ -24,13 +24,13 @@ .to belong_to(:user) .class_name('User') .with_foreign_key(:creator_id) - .inverse_of(:created_remote_development_namespace_cluster_agent_mappings) + .inverse_of(:created_namespace_cluster_agent_mappings) end it "belongs to namespace" do is_expected .to belong_to(:namespace) - .inverse_of(:remote_development_namespace_cluster_agent_mappings) + .inverse_of(:namespace_cluster_agent_mappings) end it "belongs to agent" do @@ -38,7 +38,7 @@ .to belong_to(:agent) .class_name('Clusters::Agent') .with_foreign_key(:cluster_agent_id) - .inverse_of(:remote_development_namespace_cluster_agent_mappings) + .inverse_of(:namespace_cluster_agent_mappings) end end diff --git a/ee/spec/requests/api/graphql/mutations/remote_development/namespace_cluster_agent_mapping_operations/create_spec.rb b/ee/spec/requests/api/graphql/mutations/remote_development/namespace_cluster_agent_mapping_operations/create_spec.rb index 4efe5d80a67400..0c345acdf74765 100644 --- a/ee/spec/requests/api/graphql/mutations/remote_development/namespace_cluster_agent_mapping_operations/create_spec.rb +++ b/ee/spec/requests/api/graphql/mutations/remote_development/namespace_cluster_agent_mapping_operations/create_spec.rb @@ -20,7 +20,7 @@ let(:stub_service_response) { ServiceResponse.success(payload: stub_service_payload) } let(:created_mapping) do - instance_double(RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping) + instance_double(RemoteDevelopment::NamespaceClusterAgentMapping) end let(:all_mutation_args) do -- GitLab From 70de7c3be8491355b4ccde82522a2e551f5313aa Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Thu, 20 Feb 2025 12:09:35 +0530 Subject: [PATCH 03/12] Empty TABLES_TO_BE_RENAMED the block --- lib/gitlab/database.rb | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index e14cc2e80c23ba..9eeec0482ed772 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -12,9 +12,7 @@ module Database # TABLES_TO_BE_RENAMED = { # 'old_name' => 'new_name' # }.freeze - TABLES_TO_BE_RENAMED = { - 'remote_development_namespace_cluster_agent_mappings' => 'namespace_cluster_agent_mappings' - }.freeze + TABLES_TO_BE_RENAMED = {}.freeze # Minimum PostgreSQL version requirement per documentation: # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements -- GitLab From bfe0ce5b35051de8e8c0abfc635bf59760195021 Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Thu, 20 Feb 2025 16:42:05 +0530 Subject: [PATCH 04/12] Remove the older database dictionary file --- ...velopment_namespace_cluster_agent_mappings.yml | 15 --------------- 1 file changed, 15 deletions(-) delete mode 100644 db/docs/remote_development_namespace_cluster_agent_mappings.yml diff --git a/db/docs/remote_development_namespace_cluster_agent_mappings.yml b/db/docs/remote_development_namespace_cluster_agent_mappings.yml deleted file mode 100644 index 1fdd3339692ff2..00000000000000 --- a/db/docs/remote_development_namespace_cluster_agent_mappings.yml +++ /dev/null @@ -1,15 +0,0 @@ ---- -table_name: remote_development_namespace_cluster_agent_mappings -classes: -- RemoteDevelopment::NamespaceClusterAgentMapping -feature_categories: -- workspaces -description: This table records associations between Namespaces and Cluster Agents - as a part of supporting group-cluster agent association in the context of Remote - Development Workspaces -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/145101 -milestone: '16.10' -gitlab_schema: gitlab_main_cell -sharding_key: - namespace_id: namespaces -table_size: small -- GitLab From a534c182ffb480719df8c3a0c1a46e9a536c59aa Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Thu, 20 Feb 2025 11:41:08 +0000 Subject: [PATCH 05/12] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Fabien Catteau --- ...emote_development_namespace_cluster_agent_mappings.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml index b5356054343752..c1e91326c1a355 100644 --- a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml +++ b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml @@ -2,9 +2,11 @@ view_name: remote_development_namespace_cluster_agent_mappings classes: - RemoteDevelopment::NamespaceClusterAgentMapping -feature_categories: [] -description: -introduced_by_url: '' +feature_categories: +- workspaces +description: This view replaces the remote_development_namespace_cluster_agent_mappings + table, which has been renamed to namespace_cluster_agent_mappings. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181926 gitlab_schema: gitlab_main_cell milestone: '17.10' table_size: small -- GitLab From 76634624be6d76e14b5879ab4a009f631aff5d7d Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Thu, 20 Feb 2025 11:41:16 +0000 Subject: [PATCH 06/12] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Fabien Catteau --- db/docs/namespace_cluster_agent_mappings.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/namespace_cluster_agent_mappings.yml b/db/docs/namespace_cluster_agent_mappings.yml index 2e9bb929ac7732..465929161c02fe 100644 --- a/db/docs/namespace_cluster_agent_mappings.yml +++ b/db/docs/namespace_cluster_agent_mappings.yml @@ -6,7 +6,7 @@ feature_categories: - workspaces description: This table records associations between Namespaces and Cluster Agents as a part of supporting group-cluster agent association in the context of Workspaces -introduced_by_url: '' +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181926 milestone: '17.10' gitlab_schema: gitlab_main_cell sharding_key: -- GitLab From ae21cfd2085548c731ecff997ab748c5925b5f25 Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Thu, 20 Feb 2025 19:59:30 +0530 Subject: [PATCH 07/12] Remove BBM to fix the pipeline https://gitlab.com/gitlab-org/gitlab/-/jobs/9195571526#L707 --- ..._root_namespace_cluster_agent_mappings.yml | 10 - ..._namespace_cluster_agent_mappings_again.rb | 36 --- db/schema_migrations/20240711035245 | 1 - ...l_root_namespace_cluster_agent_mappings.rb | 150 ------------- ...t_namespace_cluster_agent_mappings_spec.rb | 210 ------------------ ...l_root_namespace_cluster_agent_mappings.rb | 16 -- ...space_cluster_agent_mappings_again_spec.rb | 28 --- 7 files changed, 451 deletions(-) delete mode 100644 db/docs/batched_background_migrations/backfill_root_namespace_cluster_agent_mappings.yml delete mode 100644 db/post_migrate/20240711035245_queue_backfill_root_namespace_cluster_agent_mappings_again.rb delete mode 100644 db/schema_migrations/20240711035245 delete mode 100644 ee/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb delete mode 100644 ee/spec/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings_spec.rb delete mode 100644 lib/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb delete mode 100644 spec/migrations/20240711035245_queue_backfill_root_namespace_cluster_agent_mappings_again_spec.rb diff --git a/db/docs/batched_background_migrations/backfill_root_namespace_cluster_agent_mappings.yml b/db/docs/batched_background_migrations/backfill_root_namespace_cluster_agent_mappings.yml deleted file mode 100644 index 29551160222560..00000000000000 --- a/db/docs/batched_background_migrations/backfill_root_namespace_cluster_agent_mappings.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -migration_job_name: BackfillRootNamespaceClusterAgentMappings -description: Creates mappings between root namespaces and all cluster agents - within these namespaces that have remote development enabled -feature_category: remote_development -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/159051 -milestone: '17.2' -queued_migration_version: 20240711035245 -# Replace with the approximate date you think it's best to ensure the completion of this BBM. -finalized_by: 20241023210409 diff --git a/db/post_migrate/20240711035245_queue_backfill_root_namespace_cluster_agent_mappings_again.rb b/db/post_migrate/20240711035245_queue_backfill_root_namespace_cluster_agent_mappings_again.rb deleted file mode 100644 index ec0c4cd76b0e44..00000000000000 --- a/db/post_migrate/20240711035245_queue_backfill_root_namespace_cluster_agent_mappings_again.rb +++ /dev/null @@ -1,36 +0,0 @@ -# frozen_string_literal: true - -# See https://docs.gitlab.com/ee/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - -class QueueBackfillRootNamespaceClusterAgentMappingsAgain < Gitlab::Database::Migration[2.2] - milestone '17.2' - - # Configure the `gitlab_schema` to perform data manipulation (DML). - # Visit: https://docs.gitlab.com/ee/development/database/migrations_for_multiple_databases.html - restrict_gitlab_migration gitlab_schema: :gitlab_main_cell - MIGRATION = "BackfillRootNamespaceClusterAgentMappings" - DELAY_INTERVAL = 2.minutes - BATCH_SIZE = 1000 - SUB_BATCH_SIZE = 100 - - # Add dependent 'batched_background_migrations.queued_migration_version' values. - # DEPENDENT_BATCHED_BACKGROUND_MIGRATIONS = [] - - def up - delete_batched_background_migration(MIGRATION, :remote_development_agent_configs, :id, []) - - queue_batched_background_migration( - MIGRATION, - :remote_development_agent_configs, - :id, - job_interval: DELAY_INTERVAL, - batch_size: BATCH_SIZE, - sub_batch_size: SUB_BATCH_SIZE - ) - end - - def down - delete_batched_background_migration(MIGRATION, :remote_development_agent_configs, :id, []) - end -end diff --git a/db/schema_migrations/20240711035245 b/db/schema_migrations/20240711035245 deleted file mode 100644 index e55810988579d7..00000000000000 --- a/db/schema_migrations/20240711035245 +++ /dev/null @@ -1 +0,0 @@ -5bd102fd3ff8463e40c4467317d3f146511ef683945e7c2cabaf8bff7abd3d04 \ No newline at end of file diff --git a/ee/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb b/ee/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb deleted file mode 100644 index 2496d0026a4c6f..00000000000000 --- a/ee/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb +++ /dev/null @@ -1,150 +0,0 @@ -# frozen_string_literal: true - -# See https://docs.gitlab.com/ee/development/database/batched_background_migrations.html -# for more information on how to use batched background migrations - -# Update below commented lines with appropriate values. - -module EE - module Gitlab - module BackgroundMigration - module BackfillRootNamespaceClusterAgentMappings - extend ActiveSupport::Concern - extend ::Gitlab::Utils::Override - - prepended do - operation_name :backfill_root_namespace_cluster_agent_mappings - scope_to ->(relation) { relation.where(enabled: true) } - end - - module LogUtils - MIGRATOR = 'BackfillRootNamespaceClusterAgentMappings' - - def log_info(log_attributes) - ::Gitlab::BackgroundMigration::Logger.info(log_attributes.merge(migrator: MIGRATOR)) - end - end - - class Project < ::ApplicationRecord - self.table_name = 'projects' - end - - class Namespace < ::ApplicationRecord - self.table_name = 'namespaces' - self.inheritance_column = :_type_disabled - end - - class ClusterAgent < ::ApplicationRecord - self.table_name = 'cluster_agents' - end - - class NamespaceClusterAgentMapping < ::ApplicationRecord - extend LogUtils - - self.table_name = 'namespace_cluster_agent_mappings' - - INSERT_SQL = <<~SQL.squish - INSERT INTO namespace_cluster_agent_mappings - (namespace_id, cluster_agent_id, creator_id, created_at, updated_at) - VALUES - %{insert_tuples} - ON CONFLICT (namespace_id, cluster_agent_id) DO NOTHING; - SQL - class << self - def insert_namespace_cluster_agent_mappings(mappings) - return log_info(message: 'No mappings to create') unless mappings.present? - - tuples = mappings.map do |mapping| - format( - "(%{namespace_id}, %{cluster_agent_id}, %{creator_id}, NOW(), NOW())", - namespace_id: mapping[:namespace_id], - cluster_agent_id: mapping[:cluster_agent_id], - creator_id: migration_user.id - ) - end - - run_insert(tuples) - end - - private - - def run_insert(tuples) - insert_sql = format(INSERT_SQL, insert_tuples: tuples.join(', ')) - - connection.execute(insert_sql) - log_info(message: 'Insert query has been executed') - end - - def migration_user - ::Users::Internal.migration_bot - end - end - end - - include LogUtils - - override :perform - def perform - each_sub_batch do |sub_batch| - ids = sub_batch.map(&:id) - log_info(message: 'Migration started for batch', start_id: ids.min, end_id: ids.max) - - relevant_agent_ids = sub_batch.map(&:cluster_agent_id) - root_group_by_agent_id = root_groups_for_cluster_agents(cluster_agent_ids: relevant_agent_ids) - - mappings = root_group_by_agent_id.map do |cluster_agent_id, root_group_id| - { - namespace_id: root_group_id, - cluster_agent_id: cluster_agent_id - } - end - - NamespaceClusterAgentMapping.insert_namespace_cluster_agent_mappings(mappings) - log_info(message: 'Migration ended for batch', start_id: ids.min, end_id: ids.max) - end - end - - # rubocop:disable Metrics/AbcSize -- Disabled temporarily to roll out a time critical bugfix - def root_groups_for_cluster_agents(cluster_agent_ids:) - agents_by_id = ClusterAgent.id_in(cluster_agent_ids).index_by(&:id) - - projects_by_id = Project.id_in(agents_by_id.values.map(&:project_id)).index_by(&:id) - - project_namespaces_by_id = - Namespace.id_in(projects_by_id.values.map(&:project_namespace_id)).index_by(&:id) - - root_namespace_ids = project_namespaces_by_id.values.map do |project_namespace| - project_namespace.traversal_ids[0] - end - - root_group_namespaces_by_id = - Namespace - .id_in(root_namespace_ids) - .where(type: 'Group') - .index_by(&:id) - - cluster_agent_ids.each_with_object({}) do |cluster_agent_id, hash| - next unless agents_by_id.has_key?(cluster_agent_id) - - agent = agents_by_id[cluster_agent_id] - - # projects_by_id must contain agent.project_id as "agents" table has a ON CASCADE DELETE constraint with - # respect to the "projects" table. As such, if an agent can be retrieved from the database, - # so should its project - project = projects_by_id[agent.project_id] - - # project_namespaces_by_id must contain project.project_namespace_id as "projects" table has a - # ON CASCADE DELETE constraint with respect to the projects table. As such, if a project can be retrieved - # from the database, so should its project_namespace - project_namespace = project_namespaces_by_id[project.project_namespace_id] - - root_namespace_id = project_namespace.traversal_ids.first - - hash[cluster_agent_id] = root_namespace_id if root_group_namespaces_by_id.has_key?(root_namespace_id) - end - end - # rubocop:enable Metrics/AbcSize - end - end - end -end diff --git a/ee/spec/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings_spec.rb b/ee/spec/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings_spec.rb deleted file mode 100644 index 695602d1388d00..00000000000000 --- a/ee/spec/lib/ee/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings_spec.rb +++ /dev/null @@ -1,210 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::BackgroundMigration::BackfillRootNamespaceClusterAgentMappings, :migration, - schema: 20241023210409, feature_category: :workspaces do - let(:organizations) { table(:organizations) } - let(:namespaces) { table(:namespaces) } - let(:projects) { table(:projects) } - let(:cluster_agents) { table(:cluster_agents) } - let(:rd_agent_configs) { table(:remote_development_agent_configs) } - let(:rd_namespace_cluster_agent_mappings) { table(:namespace_cluster_agent_mappings) } - - let!(:organization) { organizations.create!(name: 'organization', path: 'organization') } - - let!(:namespace) do - namespaces.create!(name: 'root', path: 'root', type: 'Group', organization_id: organization.id).tap do |new_group| - new_group.update!(traversal_ids: [new_group.id]) - end - end - - let!(:nested_group) do - namespaces.create!( - name: 'nested', - path: 'root/nested', - type: 'Group', - organization_id: organization.id - ).tap do |new_group| - new_group.update!(traversal_ids: [namespace.id, new_group.id]) - end - end - - let!(:project) do - projects.create!( - organization_id: organization.id, - namespace_id: nested_group.id, - project_namespace_id: nested_group.id, - name: 'agent project', - path: 'agent-project' - ) - end - - let!(:agent) do - cluster_agents.create!( - name: 'agent with remote dev enabled', - project_id: project.id - ) - end - - let!(:rd_agent_config) do - rd_agent_configs.create!( - cluster_agent_id: agent.id, - enabled: true, - dns_zone: "www.example.com" - ) - end - - let(:all_rd_agent_configs) { [rd_agent_config] } - let(:migration_attrs) do - { - start_id: all_rd_agent_configs.minimum(:id), - end_id: all_rd_agent_configs.maximum(:id), - batch_table: :remote_development_agent_configs, - batch_column: :id, - sub_batch_size: 100, - pause_ms: 0, - connection: ApplicationRecord.connection - } - end - - let(:migration) { described_class.new(**migration_attrs) } - - describe '#perform' do - shared_examples "skipped migration" do - it 'skips migration for such records' do - migration.perform - - expect( - rd_namespace_cluster_agent_mappings - .where(namespace_id: namespace.id, cluster_agent_id: agent.id) - .count - ).to eq(0) - end - end - - context 'when remote dev is enabled for an agent project that resides within a group' do - it 'create a mapping from the cluster agent to its root namespace' do - migration.perform - - migrated_records = rd_namespace_cluster_agent_mappings - .where(namespace_id: namespace.id, cluster_agent_id: agent.id) - expect(migrated_records.count).to eq(1) - expect(migrated_records.first.creator_id).to eq(::Users::Internal.migration_bot.id) - end - end - - context 'when rd-enabled cluster agent project resides within a user namespace' do - let!(:namespace) do - namespaces.create!(name: 'user-namespace', path: 'user-namespace', type: 'User').tap do |new_namespace| - new_namespace.update!(traversal_ids: [new_namespace.id]) - end - end - - it_behaves_like "skipped migration" - end - - context 'when remote dev is disabled for an agent within a group' do - let!(:rd_agent_config) do - rd_agent_configs.create!( - cluster_agent_id: agent.id, - enabled: false, - dns_zone: "www.example.com" - ) - end - - it_behaves_like "skipped migration" - end - - context 'when mapping already exists between an agent and the root namespace' do - let!(:existing_mapping) do - rd_namespace_cluster_agent_mappings.create!( - cluster_agent_id: agent.id, - namespace_id: namespace.id, - creator_id: ::Users::Internal.admin_bot.id - ) - end - - it 'skips migration for such records' do - migration.perform - - mappings = rd_namespace_cluster_agent_mappings.where(namespace_id: namespace.id, cluster_agent_id: agent.id) - expect(mappings.count).to eq(1) - expect(mappings.first.creator_id).to eq(::Users::Internal.admin_bot.id) - expect(mappings.first.created_at).to be_within(0.00001.seconds).of(existing_mapping.created_at) - expect(mappings.first.updated_at).to be_within(0.00001.seconds).of(existing_mapping.updated_at) - end - end - - context 'when candidate remote dev agents exist in multiple root namespaces' do - let!(:namespace2) do - namespaces.create!(name: 'root-group-2', path: 'root2', type: 'Group').tap do |new_group| - new_group.update!(traversal_ids: [new_group.id]) - end - end - - let!(:project2) do - projects.create!( - organization_id: organization.id, - namespace_id: namespace2.id, - project_namespace_id: namespace2.id, - name: 'agent project 2', - path: 'agent-project-2' - ) - end - - let!(:agent2) do - cluster_agents.create!( - name: 'agent 2 with remote dev enabled', - project_id: project2.id - ) - end - - let!(:rd_agent_config2) do - rd_agent_configs.create!( - cluster_agent_id: agent2.id, - enabled: true, - dns_zone: "www.example2.com" - ) - end - - let!(:agent3) do - cluster_agents.create!( - name: 'agent 3 with remote dev enabled', - project_id: project.id - ) - end - - let!(:rd_agent_config3) do - rd_agent_configs.create!( - cluster_agent_id: agent3.id, - enabled: true, - dns_zone: "www.example3.com" - ) - end - - let(:all_rd_agent_configs) { [rd_agent_config, rd_agent_config2, rd_agent_config3] } - - it 'carries out migration for all agent configs' do - migration.perform - - expect(rd_namespace_cluster_agent_mappings.all.count).to eq(3) - expect( - rd_namespace_cluster_agent_mappings - .where(namespace_id: namespace2.id, cluster_agent_id: agent2.id) - .count - ).to eq(1) - expect( - rd_namespace_cluster_agent_mappings - .where(namespace_id: namespace.id, cluster_agent_id: agent.id) - .count - ).to eq(1) - expect( - rd_namespace_cluster_agent_mappings - .where(namespace_id: namespace.id, cluster_agent_id: agent3.id) - .count - ).to eq(1) - end - end - end -end diff --git a/lib/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb b/lib/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb deleted file mode 100644 index 3e143ba2534ba5..00000000000000 --- a/lib/gitlab/background_migration/backfill_root_namespace_cluster_agent_mappings.rb +++ /dev/null @@ -1,16 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module BackgroundMigration - # Backfills mappings between root namespaces and all agents within the root namespace - # that have remote development module enabled - # For more details, check: https://gitlab.com/gitlab-org/gitlab/-/issues/454411 - class BackfillRootNamespaceClusterAgentMappings < BatchedMigrationJob - feature_category :workspaces - - def perform; end - end - end -end - -Gitlab::BackgroundMigration::BackfillRootNamespaceClusterAgentMappings.prepend_mod diff --git a/spec/migrations/20240711035245_queue_backfill_root_namespace_cluster_agent_mappings_again_spec.rb b/spec/migrations/20240711035245_queue_backfill_root_namespace_cluster_agent_mappings_again_spec.rb deleted file mode 100644 index 493245824250c1..00000000000000 --- a/spec/migrations/20240711035245_queue_backfill_root_namespace_cluster_agent_mappings_again_spec.rb +++ /dev/null @@ -1,28 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' -require_migration! - -RSpec.describe QueueBackfillRootNamespaceClusterAgentMappingsAgain, migration: :gitlab_main_cell, - feature_category: :workspaces do - let!(:batched_migration) { described_class::MIGRATION } - - it 'schedules a new batched migration' do - reversible_migration do |migration| - migration.before -> { - expect(batched_migration).not_to have_scheduled_batched_migration - } - - migration.after -> { - expect(batched_migration).to have_scheduled_batched_migration( - table_name: :remote_development_agent_configs, - column_name: :id, - interval: described_class::DELAY_INTERVAL, - batch_size: described_class::BATCH_SIZE, - gitlab_schema: :gitlab_main_cell, - sub_batch_size: described_class::SUB_BATCH_SIZE - ) - } - end - end -end -- GitLab From 53ab439d038cf6d6f75d0e9d4f6a362c6789366a Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Mon, 24 Feb 2025 04:08:34 +0000 Subject: [PATCH 08/12] Apply 1 suggestion(s) to 1 file(s) --- ...rename_remote_development_namespace_cluster_agent_mapping.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb b/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb index cebb288f8401fb..869cf8fbc24711 100644 --- a/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb +++ b/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb @@ -4,7 +4,7 @@ # for more information on how to write migrations for GitLab. class RenameRemoteDevelopmentNamespaceClusterAgentMapping < Gitlab::Database::Migration[2.2] - milestone '17.9' + milestone '17.10' def up rename_table_safely(:remote_development_namespace_cluster_agent_mappings, :namespace_cluster_agent_mappings) -- GitLab From 551f0c404865851964e49bdc73b7b04ce90c15c4 Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Mon, 24 Feb 2025 12:33:10 +0530 Subject: [PATCH 09/12] Add delete view in post_migrate And remove comments from migration file. --- ..._development_namespace_cluster_agent_mapping.rb | 3 --- ...lopment_namespace_cluster_agent_mapping_view.rb | 14 ++++++++++++++ db/schema_migrations/20250224063503 | 1 + db/structure.sql | 9 --------- 4 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 db/post_migrate/20250224063503_delete_remote_development_namespace_cluster_agent_mapping_view.rb create mode 100644 db/schema_migrations/20250224063503 diff --git a/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb b/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb index 869cf8fbc24711..4d99ae53743080 100644 --- a/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb +++ b/db/migrate/20250219055827_rename_remote_development_namespace_cluster_agent_mapping.rb @@ -1,8 +1,5 @@ # frozen_string_literal: true -# See https://docs.gitlab.com/ee/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. - class RenameRemoteDevelopmentNamespaceClusterAgentMapping < Gitlab::Database::Migration[2.2] milestone '17.10' diff --git a/db/post_migrate/20250224063503_delete_remote_development_namespace_cluster_agent_mapping_view.rb b/db/post_migrate/20250224063503_delete_remote_development_namespace_cluster_agent_mapping_view.rb new file mode 100644 index 00000000000000..16e31f84d20e28 --- /dev/null +++ b/db/post_migrate/20250224063503_delete_remote_development_namespace_cluster_agent_mapping_view.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class DeleteRemoteDevelopmentNamespaceClusterAgentMappingView < Gitlab::Database::Migration[2.2] + milestone '17.10' + disable_ddl_transaction! + + def up + finalize_table_rename(:remote_development_namespace_cluster_agent_mappings, :namespace_cluster_agent_mappings) + end + + def down + undo_finalize_table_rename(:remote_development_namespace_cluster_agent_mappings, :namespace_cluster_agent_mappings) + end +end diff --git a/db/schema_migrations/20250224063503 b/db/schema_migrations/20250224063503 new file mode 100644 index 00000000000000..85f846fcfe0997 --- /dev/null +++ b/db/schema_migrations/20250224063503 @@ -0,0 +1 @@ +dbdfbe9e0140762dcb606ee83ee5ac6ee6f2f0c4b9a60979f73394a9f81ec2e6 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index ee326ee27955c7..43638c89eb53b5 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -20743,15 +20743,6 @@ CREATE SEQUENCE releases_id_seq ALTER SEQUENCE releases_id_seq OWNED BY releases.id; -CREATE VIEW remote_development_namespace_cluster_agent_mappings AS - SELECT namespace_cluster_agent_mappings.id, - namespace_cluster_agent_mappings.created_at, - namespace_cluster_agent_mappings.updated_at, - namespace_cluster_agent_mappings.namespace_id, - namespace_cluster_agent_mappings.cluster_agent_id, - namespace_cluster_agent_mappings.creator_id - FROM namespace_cluster_agent_mappings; - CREATE TABLE remote_mirrors ( id bigint NOT NULL, project_id bigint, -- GitLab From 23416b01334e0e4f1700c46c5880367b4f194ac0 Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Thu, 27 Feb 2025 17:13:29 +0530 Subject: [PATCH 10/12] Remove remote_development_namespace_cluster_agent_mappings.yml Fixes: https://gitlab.com/gitlab-org/gitlab/-/jobs/9245103629 Views that are set to be deleted in the post_migrate should not be pushed to remote repo. --- ..._development_namespace_cluster_agent_mappings.yml | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 db/docs/views/remote_development_namespace_cluster_agent_mappings.yml diff --git a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml deleted file mode 100644 index c1e91326c1a355..00000000000000 --- a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -view_name: remote_development_namespace_cluster_agent_mappings -classes: -- RemoteDevelopment::NamespaceClusterAgentMapping -feature_categories: -- workspaces -description: This view replaces the remote_development_namespace_cluster_agent_mappings - table, which has been renamed to namespace_cluster_agent_mappings. -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181926 -gitlab_schema: gitlab_main_cell -milestone: '17.10' -table_size: small -- GitLab From 3373e4eff34cbe0496856a63780c6c533d09d091 Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Thu, 27 Feb 2025 18:04:20 +0530 Subject: [PATCH 11/12] Revert "Remove remote_development_namespace_cluster_agent_mappings.yml" This reverts commit fb59b977fe70a3571ed2a2ae88fe770e81047fa8. --- ..._development_namespace_cluster_agent_mappings.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 db/docs/views/remote_development_namespace_cluster_agent_mappings.yml diff --git a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml new file mode 100644 index 00000000000000..c1e91326c1a355 --- /dev/null +++ b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml @@ -0,0 +1,12 @@ +--- +view_name: remote_development_namespace_cluster_agent_mappings +classes: +- RemoteDevelopment::NamespaceClusterAgentMapping +feature_categories: +- workspaces +description: This view replaces the remote_development_namespace_cluster_agent_mappings + table, which has been renamed to namespace_cluster_agent_mappings. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181926 +gitlab_schema: gitlab_main_cell +milestone: '17.10' +table_size: small -- GitLab From d5d864051acd512fd7018600a6e3e2d618d09406 Mon Sep 17 00:00:00 2001 From: Ashvin Sharma Date: Fri, 28 Feb 2025 12:24:57 +0530 Subject: [PATCH 12/12] Add older table entry to deleted_tables dir --- ...velopment_namespace_cluster_agent_mappings.yml | 15 +++++++++++++++ ...velopment_namespace_cluster_agent_mappings.yml | 12 ------------ 2 files changed, 15 insertions(+), 12 deletions(-) create mode 100644 db/docs/deleted_tables/remote_development_namespace_cluster_agent_mappings.yml delete mode 100644 db/docs/views/remote_development_namespace_cluster_agent_mappings.yml diff --git a/db/docs/deleted_tables/remote_development_namespace_cluster_agent_mappings.yml b/db/docs/deleted_tables/remote_development_namespace_cluster_agent_mappings.yml new file mode 100644 index 00000000000000..fcb2a5e200ba5d --- /dev/null +++ b/db/docs/deleted_tables/remote_development_namespace_cluster_agent_mappings.yml @@ -0,0 +1,15 @@ +--- +table_name: remote_development_namespace_cluster_agent_mappings +classes: + - RemoteDevelopment::RemoteDevelopmentNamespaceClusterAgentMapping +feature_categories: + - workspaces +description: Moved to deleted because it got renamed to namespace_cluster_agent_mappings +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/145101 +milestone: '16.10' +gitlab_schema: gitlab_main_cell +sharding_key: + namespace_id: namespaces +table_size: small +removed_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181926 +removed_in_milestone: '17.10' diff --git a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml b/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml deleted file mode 100644 index c1e91326c1a355..00000000000000 --- a/db/docs/views/remote_development_namespace_cluster_agent_mappings.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- -view_name: remote_development_namespace_cluster_agent_mappings -classes: -- RemoteDevelopment::NamespaceClusterAgentMapping -feature_categories: -- workspaces -description: This view replaces the remote_development_namespace_cluster_agent_mappings - table, which has been renamed to namespace_cluster_agent_mappings. -introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/181926 -gitlab_schema: gitlab_main_cell -milestone: '17.10' -table_size: small -- GitLab