diff --git a/db/database_connections/ci.yaml b/db/database_connections/ci.yaml index 4a5c2f26d951f53ce70b53360f2bb7ba19feb377..a64fa3b144a6d6d12c4252e6276b7bbe97ed961d 100644 --- a/db/database_connections/ci.yaml +++ b/db/database_connections/ci.yaml @@ -8,6 +8,7 @@ gitlab_schemas: lock_gitlab_schemas: - gitlab_main - gitlab_main_clusterwide + - gitlab_main_clusterwide_setting - gitlab_main_cell - gitlab_main_cell_local - gitlab_pm diff --git a/db/database_connections/main.yaml b/db/database_connections/main.yaml index e2184b73ce2dabe71e918a7b97ad735260fe8e0a..e652dcd8106d29ddf2e5ad67ff51e53042bf0c7b 100644 --- a/db/database_connections/main.yaml +++ b/db/database_connections/main.yaml @@ -8,6 +8,7 @@ gitlab_schemas: - gitlab_main_cell_local - gitlab_pm - gitlab_main_clusterwide + - gitlab_main_clusterwide_setting lock_gitlab_schemas: - gitlab_ci - gitlab_ci_cell_local diff --git a/db/database_connections/sec.yaml b/db/database_connections/sec.yaml index 1bc50bd46e6dfca5ed4d1ee28085fbc70455ab08..c3a8d78a6730a7aabbea1b4ce43a3d0ed7b1538a 100644 --- a/db/database_connections/sec.yaml +++ b/db/database_connections/sec.yaml @@ -7,6 +7,7 @@ gitlab_schemas: lock_gitlab_schemas: - gitlab_main - gitlab_main_clusterwide + - gitlab_main_clusterwide_setting - gitlab_main_cell - gitlab_main_cell_local - gitlab_ci diff --git a/db/docs/ai_settings.yml b/db/docs/ai_settings.yml index 989ab49e423309c2ad9603ab347e18ec75396731..b634d455bf264053c704500d6c013d5687b878a2 100644 --- a/db/docs/ai_settings.yml +++ b/db/docs/ai_settings.yml @@ -7,5 +7,5 @@ feature_categories: description: Stores instance-wide AI-related settings introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/172007 milestone: '17.6' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/docs/appearance_uploads.yml b/db/docs/appearance_uploads.yml index 451cdeec0914fcb425cb3dc192d27abe39a481c5..4310e3cbaa1d24248bfcb43f78cd8be58605ce28 100644 --- a/db/docs/appearance_uploads.yml +++ b/db/docs/appearance_uploads.yml @@ -9,4 +9,4 @@ description: Stores uploads for Appearance model introduced_by_url: milestone: '17.11' table_size: small -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting diff --git a/db/docs/appearances.yml b/db/docs/appearances.yml index cccb77fee4931eaea6788bc04b6ed843b5886393..6b1a4dc836753846373c8663d0a40f7521948f98 100644 --- a/db/docs/appearances.yml +++ b/db/docs/appearances.yml @@ -7,5 +7,5 @@ feature_categories: description: TODO introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/40104eead753e7e8ea77951a74a3941546c35aab milestone: '6.8' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/docs/application_setting_terms.yml b/db/docs/application_setting_terms.yml index 0788eb540d3c8bdbcd238ce8712dc72371dc53f5..24f92f3a36255e09fb07213eb8de6724001a6059 100644 --- a/db/docs/application_setting_terms.yml +++ b/db/docs/application_setting_terms.yml @@ -7,5 +7,5 @@ feature_categories: description: TODO introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/cf37bef287d7dd5d2dce3e2276489767b8c0671f milestone: '10.8' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/docs/application_settings.yml b/db/docs/application_settings.yml index 07dba196e35f12846c40f57db92d84bda9c40c09..92708c129b95df6453d094d6df7f06f45a8adcb3 100644 --- a/db/docs/application_settings.yml +++ b/db/docs/application_settings.yml @@ -11,5 +11,5 @@ feature_categories: description: GitLab application settings introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/8589b4e137f50293952923bb07e2814257d7784d milestone: '7.7' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/docs/broadcast_messages.yml b/db/docs/broadcast_messages.yml index c56a5a5884eac44ac113406bb42093439ee20ed6..99c5f10f11eb4ee1f86675a42c381af528f01061 100644 --- a/db/docs/broadcast_messages.yml +++ b/db/docs/broadcast_messages.yml @@ -7,5 +7,5 @@ feature_categories: description: GitLab can display broadcast messages to users of a GitLab instance introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/f1ecf53c1e55fbbc66cb2d7d12fb411cbfc2ace8 milestone: '6.3' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/docs/cloud_connector_access.yml b/db/docs/cloud_connector_access.yml index e767f2f477f83c3b253404796bde43f25f0d22a5..b0c92af8c3f567f96d9ecf55cf56c26ff70fb04f 100644 --- a/db/docs/cloud_connector_access.yml +++ b/db/docs/cloud_connector_access.yml @@ -7,5 +7,5 @@ feature_categories: description: Information about Cloud Connector features introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/140456 milestone: '16.8' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/docs/licenses.yml b/db/docs/licenses.yml index 51473ae7b6a10f329ab62b7c2bbba799bab8909e..47d32e1f7c8f3712c4e17e52e3f25e2373968809 100644 --- a/db/docs/licenses.yml +++ b/db/docs/licenses.yml @@ -7,5 +7,5 @@ feature_categories: description: Used to store information related to the instance's license introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/d1f2b09fadcfba210c5121bd214b910b9f9809fd milestone: '7.11' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/docs/plan_limits.yml b/db/docs/plan_limits.yml index 36b85305c83fb8a8b2784a18cf710d8d344e095b..bc33e532cc6e0fcbfe0e4aeadfba0e04b782c15c 100644 --- a/db/docs/plan_limits.yml +++ b/db/docs/plan_limits.yml @@ -7,5 +7,5 @@ feature_categories: description: Contains Plan specific limits (CI minute quantities for example) introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/19438 milestone: '12.5' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/docs/service_access_tokens.yml b/db/docs/service_access_tokens.yml index 1600ac70ca40f1a4ce269e4851eb6885db3f1744..552b196ca986473c30c2eb186cb5fb69ff5e8b8c 100644 --- a/db/docs/service_access_tokens.yml +++ b/db/docs/service_access_tokens.yml @@ -8,5 +8,5 @@ description: Persists JWT tokens for AI features (e.g. Code Suggestions) to auth the GitLab instance introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/125383 milestone: '16.2' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/docs/targeted_messages.yml b/db/docs/targeted_messages.yml index 2bc131f33f4d102cc89783d8779b2454d4f25e84..78d5fcbd9247352ad7f89a8e14798f3abc86dad2 100644 --- a/db/docs/targeted_messages.yml +++ b/db/docs/targeted_messages.yml @@ -7,5 +7,5 @@ feature_categories: description: In-app messages targeted at specific namespaces introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/179303 milestone: '17.10' -gitlab_schema: gitlab_main_clusterwide +gitlab_schema: gitlab_main_clusterwide_setting table_size: small diff --git a/db/gitlab_schemas/gitlab_main_clusterwide_setting.yaml b/db/gitlab_schemas/gitlab_main_clusterwide_setting.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3df133112b6bceb6643b846aa0efac58a5b75a7e --- /dev/null +++ b/db/gitlab_schemas/gitlab_main_clusterwide_setting.yaml @@ -0,0 +1,15 @@ +name: gitlab_main_clusterwide_setting +description: Schema for all Cluster setting tables, ex. application_settings, etc. +allow_cross_joins: + - gitlab_shared + - gitlab_main + - gitlab_main_cell +allow_cross_transactions: + - gitlab_internal + - gitlab_shared + - gitlab_main + - gitlab_main_cell + - gitlab_main_clusterwide +allow_cross_foreign_keys: + - gitlab_main +require_sharding_key: false diff --git a/doc/development/cells/_index.md b/doc/development/cells/_index.md index 954ae89921b2af5f5c5d89693088f07581ae4981..faf769e79a4f35b904c09bd951066f2cb661f925 100644 --- a/doc/development/cells/_index.md +++ b/doc/development/cells/_index.md @@ -14,11 +14,11 @@ Below are available schemas related to Cells and Organizations: | Schema | Description | | ------ | ----------- | | `gitlab_main_cell`| Use for all tables in the `main:` database that are for an Organization. For example, `projects` and `groups` | -| `gitlab_main_clusterwide` | All tables in the `main:` database where all rows, or a subset of rows needs to be present across the cluster, in the [Cells](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/cells/) architecture. For example, `users` and `application_settings`. For the [Cells 1.0 architecture](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/cells/iterations/cells-1.0/), there are no real clusterwide tables as each cell will have its own database. In effect, these tables will still be stored locally in each cell. | +| `gitlab_main_clusterwide_setting` | All tables in the `main:` database related to Cluster settings. For example, `application_settings`. | +| `gitlab_main_clusterwide` | All tables in the `main:` database where all rows, or a subset of rows needs to be present across the cluster, in the [Cells](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/cells/) architecture. For example, `users`. For the [Cells 1.0 architecture](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/cells/iterations/cells-1.0/), there are no real clusterwide tables as each cell will have its own database. In effect, these tables will still be stored locally in each cell. | | `gitlab_main_cell_local` | For tables in the `main:` database that are related to features that is distinct for each cell. For example, `zoekt_nodes`, or `shards`. These cell-local tables should not have any foreign key references from/to organization tables. | | `gitlab_ci` | Use for all tables in the `ci:` database that are for an Organization. For example, `ci_pipelines` and `ci_builds` | | `gitlab_ci_cell_local` | For tables in the `ci:` database that are related to features that is distinct for each cell. For example, `instance_type_ci_runners`, or `ci_cost_settings`. These cell-local tables should not have any foreign key references from/to organization tables. | -| `gitlab_main_application_setting` | Upcoming schema, related to Cluster settings. For example, `application_settings`. | | `gitlab_main_user` | Upcoming schema, related to Users. For example, `users`. | ## Choose either the `gitlab_main_cell` or `gitlab_main_clusterwide` schema diff --git a/doc/development/database/multiple_databases.md b/doc/development/database/multiple_databases.md index 3eac1fac0f28f5207061a974ab9b6eaa885bb5d1..434f9f7a5ef085278589d453761ccc88a6f90dbb 100644 --- a/doc/development/database/multiple_databases.md +++ b/doc/development/database/multiple_databases.md @@ -29,6 +29,7 @@ Each table of GitLab needs to have a `gitlab_schema` assigned: | -------- | ----------- | ------- | | `gitlab_main`| All tables that are being stored in the `main:` database. | Currently, this is being replaced with `gitlab_main_cell`, for the purpose of building the [Cells](https://handbook.gitlab.com/handbook/engineering/architecture/design-documents/cells/) architecture. | | `gitlab_main_cell` | See [Cells / Organizations schemas](../cells/_index.md#available-cells--organization-schemas) | | +| `gitlab_main_clusterwide_setting` | See [Cells / Organizations schemas](../cells/_index.md#available-cells--organization-schemas) | | | `gitlab_main_clusterwide` | See [Cells / Organizations schemas](../cells/_index.md#available-cells--organization-schemas) | | | `gitlab_main_cell_local` | See [Cells / Organizations schemas](../cells/_index.md#available-cells--organization-schemas) | | | `gitlab_ci` | All CI tables that are being stored in the `ci:` database (for example, `ci_pipelines`, `ci_builds`) | | diff --git a/lib/gitlab/database/migration_helpers.rb b/lib/gitlab/database/migration_helpers.rb index eb4fc116cebb3eacd85a53fb4b89cece2c68a222..f1632ca55c35c6e06756616700b1afc7829a85fd 100644 --- a/lib/gitlab/database/migration_helpers.rb +++ b/lib/gitlab/database/migration_helpers.rb @@ -1060,11 +1060,13 @@ def create_or_update_plan_limit(limit_name, plan_name, limit_value) plan_name_quoted = quote(plan_name) limit_value_quoted = quote(limit_value) - execute <<~SQL - INSERT INTO plan_limits (plan_id, #{limit_name_quoted}) - SELECT id, #{limit_value_quoted} FROM plans WHERE name = #{plan_name_quoted} LIMIT 1 - ON CONFLICT (plan_id) DO UPDATE SET #{limit_name_quoted} = EXCLUDED.#{limit_name_quoted}; - SQL + ::Gitlab::Database.allow_cross_joins_across_databases(url: 'https://gitlab.com/gitlab-org/gitlab/-/issues/519892') do + execute <<~SQL + INSERT INTO plan_limits (plan_id, #{limit_name_quoted}) + SELECT id, #{limit_value_quoted} FROM plans WHERE name = #{plan_name_quoted} LIMIT 1 + ON CONFLICT (plan_id) DO UPDATE SET #{limit_name_quoted} = EXCLUDED.#{limit_name_quoted}; + SQL + end end # Note this should only be used with very small tables diff --git a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb index 374fc41c99b765db25ba1eb4ae17eec77bfa964d..bf0ae20564ca7f37f438e23461f2b20bd3c27f91 100644 --- a/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb +++ b/lib/gitlab/database/query_analyzers/restrict_allowed_schemas.rb @@ -22,7 +22,8 @@ class RestrictAllowedSchemas < Base gitlab_main_clusterwide: :gitlab_main, gitlab_main_cell: :gitlab_main, gitlab_main_cell_local: :gitlab_main, - gitlab_ci_cell_local: :gitlab_ci + gitlab_ci_cell_local: :gitlab_ci, + gitlab_main_clusterwide_setting: :gitlab_main }.freeze class << self diff --git a/spec/lib/gitlab/database/dictionary_spec.rb b/spec/lib/gitlab/database/dictionary_spec.rb index ab76995dce81ea0fce8bc771337846314e235e1c..08b88d68fd7ded9a408bf7f58bb2e36bffe0180e 100644 --- a/spec/lib/gitlab/database/dictionary_spec.rb +++ b/spec/lib/gitlab/database/dictionary_spec.rb @@ -37,7 +37,7 @@ it 'returns a hash of name and schema mappings' do expect(dictionary.to_name_and_schema_mapping).to include( { - 'application_settings' => :gitlab_main_clusterwide, + 'application_settings' => :gitlab_main_clusterwide_setting, 'members' => :gitlab_main_cell } ) @@ -49,7 +49,7 @@ entry = dictionary.find_by_table_name('application_settings') expect(entry).to be_instance_of(Gitlab::Database::Dictionary::Entry) expect(entry.key_name).to eq('application_settings') - expect(entry.gitlab_schema).to eq('gitlab_main_clusterwide') + expect(entry.gitlab_schema).to eq('gitlab_main_clusterwide_setting') end it 'returns nil if the entry is not found' do @@ -106,7 +106,10 @@ describe '#name_and_schema' do it 'returns the name of the table and its gitlab schema' do - expect(database_dictionary.name_and_schema).to match_array(['application_settings', :gitlab_main_clusterwide]) + expect(database_dictionary.name_and_schema).to match_array([ + 'application_settings', + :gitlab_main_clusterwide_setting + ]) end end @@ -168,7 +171,7 @@ describe '#schema?' do it 'checks if the given schema matches the schema of the table' do expect(database_dictionary.schema?('gitlab_main')).to eq(false) - expect(database_dictionary.schema?('gitlab_main_clusterwide')).to eq(true) + expect(database_dictionary.schema?('gitlab_main_clusterwide_setting')).to eq(true) end end diff --git a/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb b/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb index cca7e5fdda3ef9ed2bf630836afb3ebe5dc4f525..40f607ddf5245e8215283a21fd5aae3abd6a0bb3 100644 --- a/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb +++ b/spec/lib/gitlab/database/no_cross_db_foreign_keys_spec.rb @@ -32,7 +32,21 @@ 'snippet_repositories.snippet_id', 'snippet_repositories.snippet_organization_id', 'snippet_repositories.snippet_project_id', - 'upload_states.upload_id' + 'upload_states.upload_id', + 'application_settings.web_ide_oauth_application_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/531355 + 'ai_settings.amazon_q_oauth_application_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/531356 + 'ai_settings.duo_workflow_oauth_application_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/531356 + 'ai_settings.duo_workflow_service_account_user_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/531356 + 'ai_settings.amazon_q_service_account_user_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/531356 + 'targeted_message_dismissals.targeted_message_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/531357 + 'user_broadcast_message_dismissals.broadcast_message_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/531358 + 'targeted_message_namespaces.targeted_message_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/531357 + 'plan_limits.plan_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/519892 + 'term_agreements.term_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/531367 + 'appearance_uploads.uploaded_by_user_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/534207 + 'appearance_uploads.project_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/534207 + 'appearance_uploads.namespace_id', # https://gitlab.com/gitlab-org/gitlab/-/issues/534207 + 'appearance_uploads.organization_id' # https://gitlab.com/gitlab-org/gitlab/-/issues/534207 ] keys << if ::Gitlab.next_rails?