diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 3651e7f6c29863a98430fde9a82f153c68807694..cf473270d6129257e0b4b2b63a0a85f2ed4387f7 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -1,5 +1,23 @@ -# Make sure that this file has the keys sorted --- +application_settings: + - table: push_rules + column: push_rule_id + on_delete: async_nullify + - table: users + column: usage_stats_set_by_user_id + on_delete: async_nullify + - table: namespaces + column: instance_administrators_group_id + on_delete: async_nullify + - table: projects + column: file_template_project_id + on_delete: async_nullify + - table: projects + column: instance_administration_project_id + on_delete: async_nullify + - table: namespaces + column: custom_project_templates_group_id + on_delete: async_nullify ci_build_report_results: - table: projects column: project_id diff --git a/db/docs/application_settings.yml b/db/docs/application_settings.yml index 1015884e8a3d073098f45ee5c6b8b64b34a29ebc..51246fcf91c19771d2b53dfc564bb81c7a77b615 100644 --- a/db/docs/application_settings.yml +++ b/db/docs/application_settings.yml @@ -11,4 +11,4 @@ feature_categories: description: GitLab application settings introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/8589b4e137f50293952923bb07e2814257d7784d milestone: '7.7' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_clusterwide diff --git a/db/docs/licenses.yml b/db/docs/licenses.yml index 0957105695e691f387b5e316fb65c0b30f3a5508..6309e7b3a9f0bdf4678f02214b043ed47e70571a 100644 --- a/db/docs/licenses.yml +++ b/db/docs/licenses.yml @@ -7,4 +7,4 @@ feature_categories: description: Used to store information related to the instance's license introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/d1f2b09fadcfba210c5121bd214b910b9f9809fd milestone: '7.11' -gitlab_schema: gitlab_main +gitlab_schema: gitlab_main_clusterwide diff --git a/db/post_migrate/20230109144915_remove_push_rules_application_settings_push_rule_id_fk.rb b/db/post_migrate/20230109144915_remove_push_rules_application_settings_push_rule_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..7ac492a6e621a72bcc98217e9437bb915ecd557c --- /dev/null +++ b/db/post_migrate/20230109144915_remove_push_rules_application_settings_push_rule_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemovePushRulesApplicationSettingsPushRuleIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :push_rules, name: "fk_693b8795e4") + + with_lock_retries do + execute('LOCK push_rules, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :push_rules, name: "fk_693b8795e4") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :push_rules, + name: "fk_693b8795e4", column: :push_rule_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109144928_remove_users_application_settings_usage_stats_set_by_user_id_fk.rb b/db/post_migrate/20230109144928_remove_users_application_settings_usage_stats_set_by_user_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..a8b9b48dbc70718a5358d2dc4ef8ab6f1427334c --- /dev/null +++ b/db/post_migrate/20230109144928_remove_users_application_settings_usage_stats_set_by_user_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveUsersApplicationSettingsUsageStatsSetByUserIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :users, name: "fk_964370041d") + + with_lock_retries do + execute('LOCK users, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :users, name: "fk_964370041d") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :users, + name: "fk_964370041d", column: :usage_stats_set_by_user_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109144941_remove_namespaces_application_settings_instance_administrators_group_id_fk.rb b/db/post_migrate/20230109144941_remove_namespaces_application_settings_instance_administrators_group_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..82f0d8a2e478d8cd55cc08b67db338b0360a9429 --- /dev/null +++ b/db/post_migrate/20230109144941_remove_namespaces_application_settings_instance_administrators_group_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveNamespacesApplicationSettingsInstanceAdministratorsGroupIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :namespaces, name: "fk_e8a145f3a7") + + with_lock_retries do + execute('LOCK namespaces, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :namespaces, name: "fk_e8a145f3a7") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :namespaces, + name: "fk_e8a145f3a7", column: :instance_administrators_group_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109144953_remove_projects_application_settings_file_template_project_id_fk.rb b/db/post_migrate/20230109144953_remove_projects_application_settings_file_template_project_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..8527930e5a74476bc69a0dc1a5818534ecd9de0b --- /dev/null +++ b/db/post_migrate/20230109144953_remove_projects_application_settings_file_template_project_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveProjectsApplicationSettingsFileTemplateProjectIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :projects, name: "fk_ec757bd087") + + with_lock_retries do + execute('LOCK projects, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :projects, name: "fk_ec757bd087") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :projects, + name: "fk_ec757bd087", column: :file_template_project_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109145005_remove_projects_application_settings_instance_administration_project_id_fk.rb b/db/post_migrate/20230109145005_remove_projects_application_settings_instance_administration_project_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..e91769a90bb4b142bd060bf57122a7d6aed341a9 --- /dev/null +++ b/db/post_migrate/20230109145005_remove_projects_application_settings_instance_administration_project_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveProjectsApplicationSettingsInstanceAdministrationProjectIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :projects, name: "fk_rails_7e112a9599") + + with_lock_retries do + execute('LOCK projects, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :projects, name: "fk_rails_7e112a9599") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :projects, + name: "fk_rails_7e112a9599", column: :instance_administration_project_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/post_migrate/20230109145018_remove_namespaces_application_settings_custom_project_templates_group_id_fk.rb b/db/post_migrate/20230109145018_remove_namespaces_application_settings_custom_project_templates_group_id_fk.rb new file mode 100644 index 0000000000000000000000000000000000000000..cfc915f57b45601465488510323a9135dd0c6569 --- /dev/null +++ b/db/post_migrate/20230109145018_remove_namespaces_application_settings_custom_project_templates_group_id_fk.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class RemoveNamespacesApplicationSettingsCustomProjectTemplatesGroupIdFk < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + return unless foreign_key_exists?(:application_settings, :namespaces, name: "fk_rails_b53e481273") + + with_lock_retries do + execute('LOCK namespaces, application_settings IN ACCESS EXCLUSIVE MODE') if transaction_open? + + remove_foreign_key_if_exists(:application_settings, :namespaces, name: "fk_rails_b53e481273") + end + end + + def down + add_concurrent_foreign_key(:application_settings, :namespaces, + name: "fk_rails_b53e481273", column: :custom_project_templates_group_id, + target_column: :id, on_delete: :nullify) + end +end diff --git a/db/schema_migrations/20230109144915 b/db/schema_migrations/20230109144915 new file mode 100644 index 0000000000000000000000000000000000000000..3aa06a8e06cc3d1485a93ed02f198f5930e70efa --- /dev/null +++ b/db/schema_migrations/20230109144915 @@ -0,0 +1 @@ +af5a388ec9716c4ad7e1b513c672aee382e56f859bfbc5b4bd238a6d3c3c2155 \ No newline at end of file diff --git a/db/schema_migrations/20230109144928 b/db/schema_migrations/20230109144928 new file mode 100644 index 0000000000000000000000000000000000000000..e8491d35e6ffb70b48887d37d0963b84f5b64aa1 --- /dev/null +++ b/db/schema_migrations/20230109144928 @@ -0,0 +1 @@ +1eb5fb28c64d25991587b61821293bb94216a7753b3444568ab389cd1fbfe416 \ No newline at end of file diff --git a/db/schema_migrations/20230109144941 b/db/schema_migrations/20230109144941 new file mode 100644 index 0000000000000000000000000000000000000000..a9ddefa2314a447bc340eed950403d2e976845b5 --- /dev/null +++ b/db/schema_migrations/20230109144941 @@ -0,0 +1 @@ +9a10f3fdd239fb65b36d9820474adbf804a2f93f14a11b33007946872e637e36 \ No newline at end of file diff --git a/db/schema_migrations/20230109144953 b/db/schema_migrations/20230109144953 new file mode 100644 index 0000000000000000000000000000000000000000..b677a0cba24c306dd74947b92320aa7fa99fe123 --- /dev/null +++ b/db/schema_migrations/20230109144953 @@ -0,0 +1 @@ +da3526f94be5d3ae58c2f12efc725cc9782d6611f4c75cdb02af501eb97437a7 \ No newline at end of file diff --git a/db/schema_migrations/20230109145005 b/db/schema_migrations/20230109145005 new file mode 100644 index 0000000000000000000000000000000000000000..703e57b88c572657065085caa0435ce467b7bf76 --- /dev/null +++ b/db/schema_migrations/20230109145005 @@ -0,0 +1 @@ +5d094f7f2425ce48874af990cdda371da5956c70c4c520c85ba20ce243518b70 \ No newline at end of file diff --git a/db/schema_migrations/20230109145018 b/db/schema_migrations/20230109145018 new file mode 100644 index 0000000000000000000000000000000000000000..0d89853b93129f55f448e0521584b2962c463fc1 --- /dev/null +++ b/db/schema_migrations/20230109145018 @@ -0,0 +1 @@ +7285f96cefaecee6fb83d16bfa0559c967cb28269a9ebaf806e16c5c9850c7fb \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index c30bfac2e9e9c4b7a1f396d73fd1fb3ec66b697f..878789a0cf8264abbe870524baa3a2c80e0a5c60 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -33519,9 +33519,6 @@ ALTER TABLE ONLY merge_requests ALTER TABLE ONLY ci_builds ADD CONSTRAINT fk_6661f4f0e8 FOREIGN KEY (resource_group_id) REFERENCES ci_resource_groups(id) ON DELETE SET NULL; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_693b8795e4 FOREIGN KEY (push_rule_id) REFERENCES push_rules(id) ON DELETE SET NULL; - ALTER TABLE ONLY project_wiki_repository_states ADD CONSTRAINT fk_6951681c70 FOREIGN KEY (project_wiki_repository_id) REFERENCES project_wiki_repositories(id) ON DELETE CASCADE; @@ -33684,9 +33681,6 @@ ALTER TABLE ONLY vulnerabilities ALTER TABLE ONLY boards_epic_list_user_preferences ADD CONSTRAINT fk_95eac55851 FOREIGN KEY (epic_list_id) REFERENCES boards_epic_lists(id) ON DELETE CASCADE; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_964370041d FOREIGN KEY (usage_stats_set_by_user_id) REFERENCES users(id) ON DELETE SET NULL; - ALTER TABLE ONLY issues ADD CONSTRAINT fk_96b1dd429c FOREIGN KEY (milestone_id) REFERENCES milestones(id) ON DELETE SET NULL; @@ -34026,18 +34020,12 @@ ALTER TABLE ONLY namespaces ALTER TABLE ONLY fork_networks ADD CONSTRAINT fk_e7b436b2b5 FOREIGN KEY (root_project_id) REFERENCES projects(id) ON DELETE SET NULL; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_e8a145f3a7 FOREIGN KEY (instance_administrators_group_id) REFERENCES namespaces(id) ON DELETE SET NULL; - ALTER TABLE ONLY integrations ADD CONSTRAINT fk_e8fe908a34 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; ALTER TABLE ONLY pages_domains ADD CONSTRAINT fk_ea2f6dfc6f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_ec757bd087 FOREIGN KEY (file_template_project_id) REFERENCES projects(id) ON DELETE SET NULL; - ALTER TABLE ONLY merge_requests_compliance_violations ADD CONSTRAINT fk_ec881c1c6f FOREIGN KEY (violating_user_id) REFERENCES users(id) ON DELETE CASCADE; @@ -34965,9 +34953,6 @@ ALTER TABLE ONLY milestone_releases ALTER TABLE ONLY resource_state_events ADD CONSTRAINT fk_rails_7ddc5f7457 FOREIGN KEY (source_merge_request_id) REFERENCES merge_requests(id) ON DELETE SET NULL; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_rails_7e112a9599 FOREIGN KEY (instance_administration_project_id) REFERENCES projects(id) ON DELETE SET NULL; - ALTER TABLE ONLY clusters_kubernetes_namespaces ADD CONSTRAINT fk_rails_7e7688ecaf FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE; @@ -35322,9 +35307,6 @@ ALTER TABLE ONLY merge_request_predictions ALTER TABLE ONLY incident_management_escalation_rules ADD CONSTRAINT fk_rails_b3c9c17bd4 FOREIGN KEY (oncall_schedule_id) REFERENCES incident_management_oncall_schedules(id) ON DELETE CASCADE; -ALTER TABLE ONLY application_settings - ADD CONSTRAINT fk_rails_b53e481273 FOREIGN KEY (custom_project_templates_group_id) REFERENCES namespaces(id) ON DELETE SET NULL; - ALTER TABLE ONLY packages_debian_project_component_files ADD CONSTRAINT fk_rails_b543a9622b FOREIGN KEY (architecture_id) REFERENCES packages_debian_project_architectures(id) ON DELETE RESTRICT; diff --git a/lib/gitlab/database.rb b/lib/gitlab/database.rb index 40e2e6371149119ebf68828dba2460c66b7f578a..e750edc393f6dd83549c51128d23c83412e6e070 100644 --- a/lib/gitlab/database.rb +++ b/lib/gitlab/database.rb @@ -102,7 +102,8 @@ def self.schemas_to_base_models gitlab_ci: [self.database_base_models[:ci] || self.database_base_models.fetch(:main)], # use CI or fallback to main gitlab_shared: database_base_models_with_gitlab_shared.values, # all models gitlab_internal: database_base_models.values, # all models - gitlab_pm: [self.database_base_models.fetch(:main)] # package metadata models + gitlab_pm: [self.database_base_models.fetch(:main)], # package metadata models + gitlab_main_clusterwide: [self.database_base_models.fetch(:main)] }.with_indifferent_access.freeze end diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index c51282c9a553bd1557c96965fd301401101506a8..4ae3622479f0800c0553959e132143ec9051898e 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -9,7 +9,18 @@ class RestrictAllowedSchemas < Base DMLNotAllowedError = Class.new(UnsupportedSchemaError) DMLAccessDeniedError = Class.new(UnsupportedSchemaError) - IGNORED_SCHEMAS = %i[gitlab_shared gitlab_internal].freeze + # Re-map schemas observed schemas to a single cluster mode + # - symbol: + # The mapped schema indicates that it contains all data in a single-cluster mode + # - nil: + # Inidicates that changes made to this schema are ignored and always allowed + SCHEMA_MAPPING = { + gitlab_shared: nil, + gitlab_internal: nil, + + # Pods specific changes + gitlab_main_clusterwide: :gitlab_main + }.freeze class << self def enabled? @@ -90,7 +101,13 @@ def dml_tables(parsed) def dml_schemas(tables) extra_schemas = ::Gitlab::Database::GitlabSchema.table_schemas(tables) - extra_schemas.subtract(IGNORED_SCHEMAS) + + SCHEMA_MAPPING.each do |schema, mapped_schema| + next unless extra_schemas.delete?(schema) + + extra_schemas.add(mapped_schema) if mapped_schema + end + extra_schemas end diff --git a/scripts/decomposition/generate-loose-foreign-key b/scripts/decomposition/generate-loose-foreign-key index 2e91aa662dee95de8687f65a7b636b33c7bda6a4..ad7d6e32aa0201f549e944a88b9362a506cf2e8c 100755 --- a/scripts/decomposition/generate-loose-foreign-key +++ b/scripts/decomposition/generate-loose-foreign-key @@ -164,7 +164,7 @@ def generate_migration(definition) content = <<-EOF.strip_heredoc # frozen_string_literal: true - class Remove#{definition.to_table.camelcase}#{definition.from_table.camelcase}#{definition.column.camelcase}Fk < Gitlab::Database::Migration[1.0] + class Remove#{definition.to_table.camelcase}#{definition.from_table.camelcase}#{definition.column.camelcase}Fk < Gitlab::Database::Migration[2.1] disable_ddl_transaction! def up @@ -178,7 +178,9 @@ def generate_migration(definition) end def down - add_concurrent_foreign_key(:#{definition.from_table}, :#{definition.to_table}, name: "#{definition.name}", column: :#{definition.column}, target_column: :#{definition.primary_key}, on_delete: :#{definition.on_delete}) + add_concurrent_foreign_key(:#{definition.from_table}, :#{definition.to_table}, + name: "#{definition.name}", column: :#{definition.column}, + target_column: :#{definition.primary_key}, on_delete: :#{definition.on_delete}) end end EOF diff --git a/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb b/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb index d67e50a50d41a2aa254ae19f6337afbaf70265eb..d878d46c883fb516df9ef551da7c25e82f3cf945 100644 --- a/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb +++ b/spec/lib/gitlab/database_importers/self_monitoring/project/delete_service_spec.rb @@ -38,6 +38,8 @@ it 'deletes project ID from application settings' do subject.execute + LooseForeignKeys::ProcessDeletedRecordsService.new(connection: Project.connection).execute + expect(application_setting.reload.self_monitoring_project_id).to be_nil end