From 763ca76881c8c54e4771abb860c4e2ae71e51779 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 9 Jan 2023 15:38:22 +0100 Subject: [PATCH 1/3] Pods: Make Application Settings to be part of `gitlab_main_cluster` This disconnects `application_settings` from Pod-local database. Changelog: changed --- db/docs/application_settings.yml | 2 +- lib/gitlab/database.rb | 3 ++- .../restrict_allowed_schemas.rb | 21 +++++++++++++++++-- 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/db/docs/application_settings.yml b/db/docs/application_settings.yml index 1015884e8a3d07..51246fcf91c197 100644 --- a/db/docs/application_settings.yml +++ b/db/docs/application_settings.yml @@ -11,4 +11,4 @@ feature_categories: description: GitLab application settings introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/8589b4e137f50293952923bb07e2814257d7784d milestone: '7.7' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_clusterwide diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 40e2e637114911..e750edc393f6dd 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -102,7 +102,8 @@ def self.schemas_to_base_models gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main gitlab_shared: database_base_models_with_gitlab_shared.values, # all models gitlab_internal: database_base_models.values, # all models - gitlab_pm: [self.database_base_models.fetch(:main)] # package metadata models + gitlab_pm: [self.database_base_models.fetch(:main)], # package metadata models + gitlab_main_clusterwide: [self.database_base_models.fetch(:main)] }.with_indifferent_access.freeze end diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index c51282c9a553bd..4ae3622479f080 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -9,7 +9,18 @@ class RestrictAllowedSchemas < Base DMLNotAllowedError = Class.new(UnsupportedSchemaError) DMLAccessDeniedError = Class.new(UnsupportedSchemaError) - IGNORED_SCHEMAS = %i[gitlab_shared gitlab_internal].freeze + # Re-map schemas observed schemas to a single cluster mode + # - symbol: + # The mapped schema indicates that it contains all data in a single-cluster mode + # - nil: + # Inidicates that changes made to this schema are ignored and always allowed + SCHEMA_MAPPING = { + gitlab_shared: nil, + gitlab_internal: nil, + + # Pods specific changes + gitlab_main_clusterwide: :gitlab_main + }.freeze class << self def enabled? @@ -90,7 +101,13 @@ def dml_tables(parsed) def dml_schemas(tables) extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) - extra_schemas.subtract(IGNORED_SCHEMAS) + + SCHEMA_MAPPING.each do |schema, mapped_schema| + next unless extra_schemas.delete?(schema) + + extra_schemas.add(mapped_schema) if mapped_schema + end + extra_schemas end -- GitLab From 7ed45214a24c68a0a890decf878e837f6e808fea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Tue, 17 Jan 2023 16:19:15 +0100 Subject: [PATCH 2/3] Pods: Make `licenses` to be `clusterwide` --- db/docs/licenses.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/licenses.yml b/db/docs/licenses.yml index 0957105695e691..6309e7b3a9f0bd 100644 --- a/db/docs/licenses.yml +++ b/db/docs/licenses.yml @@ -7,4 +7,4 @@ feature_categories: description: Used to store information related to the instance's license introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/d1f2b09fadcfba210c5121bd214b910b9f9809fd milestone: '7.11' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_clusterwide -- GitLab From 197c627e261c3f6949bdb95dc7568bae70c470fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Mon, 9 Jan 2023 15:52:18 +0100 Subject: [PATCH 3/3] Remove foreign keys to `application_settings` Since `application_settings` is part of `gitlab_main_cluster` reduce foregin keys to `gitlab_main`. --- config/gitlab_loose_foreign_keys.yml | 20 +++++++++++++++++- ...es_application_settings_push_rule_id_fk.rb | 21 +++++++++++++++++++ ..._settings_usage_stats_set_by_user_id_fk.rb | 21 +++++++++++++++++++ ...ngs_instance_administrators_group_id_fk.rb | 21 +++++++++++++++++++ ...on_settings_file_template_project_id_fk.rb | 21 +++++++++++++++++++ ...s_instance_administration_project_id_fk.rb | 21 +++++++++++++++++++ ...gs_custom_project_templates_group_id_fk.rb | 21 +++++++++++++++++++ db/schema_migrations/20230109144915 | 1 + db/schema_migrations/20230109144928 | 1 + db/schema_migrations/20230109144941 | 1 + db/schema_migrations/20230109144953 | 1 + db/schema_migrations/20230109145005 | 1 + db/schema_migrations/20230109145018 | 1 + db/structure.sql | 18 ---------------- .../decomposition/generate-loose-foreign-key | 6 ++++-- .../project/delete_service_spec.rb | 2 ++ 16 files changed, 157 insertions(+), 21 deletions(-) create mode 100644 db/post_migrate/20230109144915_remove_push_rules_application_settings_push_rule_id_fk.rb create mode 100644 db/post_migrate/20230109144928_remove_users_application_settings_usage_stats_set_by_user_id_fk.rb create mode 100644 db/post_migrate/20230109144941_remove_namespaces_application_settings_instance_administrators_group_id_fk.rb create mode 100644 db/post_migrate/20230109144953_remove_projects_application_settings_file_template_project_id_fk.rb create mode 100644 db/post_migrate/20230109145005_remove_projects_application_settings_instance_administration_project_id_fk.rb create mode 100644 db/post_migrate/20230109145018_remove_namespaces_application_settings_custom_project_templates_group_id_fk.rb create mode 100644 db/schema_migrations/20230109144915 create mode 100644 db/schema_migrations/20230109144928 create mode 100644 db/schema_migrations/20230109144941 create mode 100644 db/schema_migrations/20230109144953 create mode 100644 db/schema_migrations/20230109145005 create mode 100644 db/schema_migrations/20230109145018 diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 3651e7f6c29863..cf473270d61292 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -1,5 +1,23 @@ -# Make sure that this file has the keys sorted --- +application_settings: + - table: push_rules + column: push_rule_id + on_delete: async_nullify + - table: users + column: usage_stats_set_by_user_id + on_delete: async_nullify + - table: namespaces + column: instance_administrators_group_id + on_delete: async_nullify + - table: projects + column: file_template_project_id + on_delete: async_nullify + - table: projects + column: instance_administration_project_id + on_delete: async_nullify + - table: namespaces + column: custom_project_templates_group_id + on_delete: async_nullify ci_build_report_results: - table: projects column: project_id diff --git a/db/post_migrate/20230109144915_remove_push_rules_application_settings_push_rule_id_fk.rb b/db/post_migrate/20230109144915_remove_push_rules_application_settings_push_rule_id_fk.rb new file mode 100644 index 00000000000000..7ac492a6e621a7 --- /dev/null +++ b/db/post_migrate/20230109144915_remove_push_rules_application_settings_push_rule_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemovePushRulesApplicationSettingsPushRuleIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :push_rules, name: "fk_693b8795e4") + + with_lock_retries do + execute('LOCK push_rules, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :push_rules, name: "fk_693b8795e4") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :push_rules, + name: "fk_693b8795e4", column: :push_rule_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109144928_remove_users_application_settings_usage_stats_set_by_user_id_fk.rb b/db/post_migrate/20230109144928_remove_users_application_settings_usage_stats_set_by_user_id_fk.rb new file mode 100644 index 00000000000000..a8b9b48dbc7071 --- /dev/null +++ b/db/post_migrate/20230109144928_remove_users_application_settings_usage_stats_set_by_user_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveUsersApplicationSettingsUsageStatsSetByUserIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :users, name: "fk_964370041d") + + with_lock_retries do + execute('LOCK users, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :users, name: "fk_964370041d") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :users, + name: "fk_964370041d", column: :usage_stats_set_by_user_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109144941_remove_namespaces_application_settings_instance_administrators_group_id_fk.rb b/db/post_migrate/20230109144941_remove_namespaces_application_settings_instance_administrators_group_id_fk.rb new file mode 100644 index 00000000000000..82f0d8a2e478d8 --- /dev/null +++ b/db/post_migrate/20230109144941_remove_namespaces_application_settings_instance_administrators_group_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveNamespacesApplicationSettingsInstanceAdministratorsGroupIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :namespaces, name: "fk_e8a145f3a7") + + with_lock_retries do + execute('LOCK namespaces, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :namespaces, name: "fk_e8a145f3a7") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :namespaces, + name: "fk_e8a145f3a7", column: :instance_administrators_group_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109144953_remove_projects_application_settings_file_template_project_id_fk.rb b/db/post_migrate/20230109144953_remove_projects_application_settings_file_template_project_id_fk.rb new file mode 100644 index 00000000000000..8527930e5a7447 --- /dev/null +++ b/db/post_migrate/20230109144953_remove_projects_application_settings_file_template_project_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveProjectsApplicationSettingsFileTemplateProjectIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :projects, name: "fk_ec757bd087") + + with_lock_retries do + execute('LOCK projects, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :projects, name: "fk_ec757bd087") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :projects, + name: "fk_ec757bd087", column: :file_template_project_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109145005_remove_projects_application_settings_instance_administration_project_id_fk.rb b/db/post_migrate/20230109145005_remove_projects_application_settings_instance_administration_project_id_fk.rb new file mode 100644 index 00000000000000..e91769a90bb4b1 --- /dev/null +++ b/db/post_migrate/20230109145005_remove_projects_application_settings_instance_administration_project_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveProjectsApplicationSettingsInstanceAdministrationProjectIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :projects, name: "fk_rails_7e112a9599") + + with_lock_retries do + execute('LOCK projects, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :projects, name: "fk_rails_7e112a9599") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :projects, + name: "fk_rails_7e112a9599", column: :instance_administration_project_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109145018_remove_namespaces_application_settings_custom_project_templates_group_id_fk.rb b/db/post_migrate/20230109145018_remove_namespaces_application_settings_custom_project_templates_group_id_fk.rb new file mode 100644 index 00000000000000..cfc915f57b4560 --- /dev/null +++ b/db/post_migrate/20230109145018_remove_namespaces_application_settings_custom_project_templates_group_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveNamespacesApplicationSettingsCustomProjectTemplatesGroupIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :namespaces, name: "fk_rails_b53e481273") + + with_lock_retries do + execute('LOCK namespaces, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :namespaces, name: "fk_rails_b53e481273") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :namespaces, + name: "fk_rails_b53e481273", column: :custom_project_templates_group_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/schema_migrations/20230109144915 b/db/schema_migrations/20230109144915 new file mode 100644 index 00000000000000..3aa06a8e06cc3d --- /dev/null +++ b/db/schema_migrations/20230109144915 @@ -0,0 +1 @@ +af5a388ec9716c4ad7e1b513c672aee382e56f859bfbc5b4bd238a6d3c3c2155 \ No newline at end of file diff --git a/db/schema_migrations/20230109144928 b/db/schema_migrations/20230109144928 new file mode 100644 index 00000000000000..e8491d35e6ffb7 --- /dev/null +++ b/db/schema_migrations/20230109144928 @@ -0,0 +1 @@ +1eb5fb28c64d25991587b61821293bb94216a7753b3444568ab389cd1fbfe416 \ No newline at end of file diff --git a/db/schema_migrations/20230109144941 b/db/schema_migrations/20230109144941 new file mode 100644 index 00000000000000..a9ddefa2314a44 --- /dev/null +++ b/db/schema_migrations/20230109144941 @@ -0,0 +1 @@ +9a10f3fdd239fb65b36d9820474adbf804a2f93f14a11b33007946872e637e36 \ No newline at end of file diff --git a/db/schema_migrations/20230109144953 b/db/schema_migrations/20230109144953 new file mode 100644 index 00000000000000..b677a0cba24c30 --- /dev/null +++ b/db/schema_migrations/20230109144953 @@ -0,0 +1 @@ +da3526f94be5d3ae58c2f12efc725cc9782d6611f4c75cdb02af501eb97437a7 \ No newline at end of file diff --git a/db/schema_migrations/20230109145005 b/db/schema_migrations/20230109145005 new file mode 100644 index 00000000000000..703e57b88c5726 --- /dev/null +++ b/db/schema_migrations/20230109145005 @@ -0,0 +1 @@ +5d094f7f2425ce48874af990cdda371da5956c70c4c520c85ba20ce243518b70 \ No newline at end of file diff --git a/db/schema_migrations/20230109145018 b/db/schema_migrations/20230109145018 new file mode 100644 index 00000000000000..0d89853b93129f --- /dev/null +++ b/db/schema_migrations/20230109145018 @@ -0,0 +1 @@ +7285f96cefaecee6fb83d16bfa0559c967cb28269a9ebaf806e16c5c9850c7fb \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index c30bfac2e9e9c4..878789a0cf8264 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -33519,9 +33519,6 @@ ALTER TABLE ONLY merge_requests ALTER TABLE ONLY ci_builds ADD CONSTRAINT fk_6661f4f0e8 FOREIGN KEY (resource_group_id) REFERENCES ci_resource_groups(id) ON DELETE SET NULL; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_693b8795e4 FOREIGN KEY (push_rule_id) REFERENCES push_rules(id) ON DELETE SET NULL; - ALTER TABLE ONLY project_wiki_repository_states ADD CONSTRAINT fk_6951681c70 FOREIGN KEY (project_wiki_repository_id) REFERENCES project_wiki_repositories(id) ON DELETE CASCADE; @@ -33684,9 +33681,6 @@ ALTER TABLE ONLY vulnerabilities ALTER TABLE ONLY boards_epic_list_user_preferences ADD CONSTRAINT fk_95eac55851 FOREIGN KEY (epic_list_id) REFERENCES boards_epic_lists(id) ON DELETE CASCADE; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_964370041d FOREIGN KEY (usage_stats_set_by_user_id) REFERENCES users(id) ON DELETE SET NULL; - ALTER TABLE ONLY issues ADD CONSTRAINT fk_96b1dd429c FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE SET NULL; @@ -34026,18 +34020,12 @@ ALTER TABLE ONLY namespaces ALTER TABLE ONLY fork_networks ADD CONSTRAINT fk_e7b436b2b5 FOREIGN KEY (root_project_id) REFERENCES projects(id) ON DELETE SET NULL; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_e8a145f3a7 FOREIGN KEY (instance_administrators_group_id) REFERENCES namespaces(id) ON DELETE SET NULL; - ALTER TABLE ONLY integrations ADD CONSTRAINT fk_e8fe908a34 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; ALTER TABLE ONLY pages_domains ADD CONSTRAINT fk_ea2f6dfc6f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_ec757bd087 FOREIGN KEY (file_template_project_id) REFERENCES projects(id) ON DELETE SET NULL; - ALTER TABLE ONLY merge_requests_compliance_violations ADD CONSTRAINT fk_ec881c1c6f FOREIGN KEY (violating_user_id) REFERENCES users(id) ON DELETE CASCADE; @@ -34965,9 +34953,6 @@ ALTER TABLE ONLY milestone_releases ALTER TABLE ONLY resource_state_events ADD CONSTRAINT fk_rails_7ddc5f7457 FOREIGN KEY (source_merge_request_id) REFERENCES merge_requests(id) ON DELETE SET NULL; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_rails_7e112a9599 FOREIGN KEY (instance_administration_project_id) REFERENCES projects(id) ON DELETE SET NULL; - ALTER TABLE ONLY clusters_kubernetes_namespaces ADD CONSTRAINT fk_rails_7e7688ecaf FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE; @@ -35322,9 +35307,6 @@ ALTER TABLE ONLY merge_request_predictions ALTER TABLE ONLY incident_management_escalation_rules ADD CONSTRAINT fk_rails_b3c9c17bd4 FOREIGN KEY (oncall_schedule_id) REFERENCES incident_management_oncall_schedules(id) ON DELETE CASCADE; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_rails_b53e481273 FOREIGN KEY (custom_project_templates_group_id) REFERENCES namespaces(id) ON DELETE SET NULL; - ALTER TABLE ONLY packages_debian_project_component_files ADD CONSTRAINT fk_rails_b543a9622b FOREIGN KEY (architecture_id) REFERENCES packages_debian_project_architectures(id) ON DELETE RESTRICT; diff --git a/scripts/decomposition/generate-loose-foreign-key b/scripts/decomposition/generate-loose-foreign-key index 2e91aa662dee95..ad7d6e32aa0201 100755 --- a/scripts/decomposition/generate-loose-foreign-key +++ b/scripts/decomposition/generate-loose-foreign-key @@ -164,7 +164,7 @@ def generate_migration(definition) content = <<-EOF.strip_heredoc # frozen_string_literal: true - class Remove#{definition.to_table.camelcase}#{definition.from_table.camelcase}#{definition.column.camelcase}Fk < Gitlab::Database::Migration[1.0] + class Remove#{definition.to_table.camelcase}#{definition.from_table.camelcase}#{definition.column.camelcase}Fk < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -178,7 +178,9 @@ def generate_migration(definition) end def down - add_concurrent_foreign_key(:#{definition.from_table}, :#{definition.to_table}, name: "#{definition.name}", column: :#{definition.column}, target_column: :#{definition.primary_key}, on_delete: :#{definition.on_delete}) + add_concurrent_foreign_key(:#{definition.from_table}, :#{definition.to_table}, + name: "#{definition.name}", column: :#{definition.column}, + target_column: :#{definition.primary_key}, on_delete: :#{definition.on_delete}) end end EOF diff --git a/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb b/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb index d67e50a50d41a2..d878d46c883fb5 100644 --- a/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb +++ b/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb @@ -38,6 +38,8 @@ it 'deletes project ID from application settings' do subject.execute + LooseForeignKeys::ProcessDeletedRecordsService.new(connection: Project.connection).execute + expect(application_setting.reload.self_monitoring_project_id).to be_nil end -- GitLab