From eb873812066a3464596e29909e186b6cffc11895 Mon Sep 17 00:00:00 2001 From: Matt Kasa Date: Thu, 20 Apr 2023 00:12:59 -0700 Subject: [PATCH] Add support for the embedding database Also: - Moves geo database docs to ee/db/geo/docs/ - Adds support for embedding database docs in ee/db/embedding/docs/ Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/404396 --- ...Geo Replicate a new Git repository type.md | 2 +- .../Geo Replicate a new blob type.md | 2 +- .rubocop.yml | 1 + .../active_record_database_tasks.rb | 2 +- db/docs/schema_migrations.yml | 5 +- .../database/database_dictionary.md | 15 +++-- ee/app/models/embedding/application_record.rb | 21 +++++++ ee/db/embedding/docs/ar_internal_metadata.yml | 11 ++++ ee/db/embedding/docs/schema_migrations.yml | 13 +++++ ee/db/embedding/migrate/.gitkeep | 0 .../20230419232013_create_vector_extension.rb | 13 +++++ ee/db/embedding/post_migrate/.gitkeep | 0 ee/db/embedding/schema_migrations/.gitkeep | 0 .../schema_migrations/20230419232013 | 1 + ee/db/embedding/seeds.rb | 0 ee/db/embedding/structure.sql | 18 ++++++ ee/db/{ => geo}/docs/ar_internal_metadata.yml | 0 .../docs/ci_secure_file_registry.yml | 0 .../docs/container_repository_registry.yml | 0 .../docs/dependency_proxy_blob_registry.yml | 0 .../dependency_proxy_manifest_registry.yml | 0 ee/db/{ => geo}/docs/design_registry.yml | 0 ee/db/{ => geo}/docs/event_log_states.yml | 0 ee/db/{ => geo}/docs/file_registry.yml | 0 .../docs/group_wiki_repository_registry.yml | 0 .../{ => geo}/docs/job_artifact_registry.yml | 0 ee/db/{ => geo}/docs/lfs_object_registry.yml | 0 .../docs/merge_request_diff_registry.yml | 0 .../{ => geo}/docs/package_file_registry.yml | 0 .../docs/pages_deployment_registry.yml | 0 .../docs/pipeline_artifact_registry.yml | 0 ee/db/{ => geo}/docs/project_registry.yml | 0 .../docs/project_wiki_repository_registry.yml | 0 ee/db/{ => geo}/docs/schema_migrations.yml | 7 ++- ee/db/{ => geo}/docs/secondary_usage_data.yml | 0 .../docs/snippet_repository_registry.yml | 0 .../docs/terraform_state_version_registry.yml | 0 ee/lib/ee/gitlab/database.rb | 24 ++++++-- ee/lib/ee/gitlab/database/gitlab_schema.rb | 18 ++++-- ..._tasks.rb => additional_database_tasks.rb} | 12 ++-- ee/lib/tasks/embedding.rake | 20 +++++++ .../lib/ee/gitlab/database/docs/docs_spec.rb | 46 +++++++++++---- .../ee/gitlab/database/gitlab_schema_spec.rb | 5 +- ...c.rb => additional_database_tasks_spec.rb} | 58 ++++++++++--------- .../embedding/application_record_spec.rb | 13 +++++ .../models/embedding/schema_migration_spec.rb | 15 +++++ .../support/helpers/ee/require_migration.rb | 7 ++- lib/gitlab/database/gitlab_schema.rb | 4 ++ lib/gitlab/database/tables_locker.rb | 2 +- lib/gitlab/patch/database_config.rb | 18 +++--- lib/tasks/gitlab/db.rake | 17 ++++-- rubocop/migration_helpers.rb | 4 +- .../lib/gitlab/database/gitlab_schema_spec.rb | 6 +- spec/tasks/gitlab/db_rake_spec.rb | 1 + 54 files changed, 293 insertions(+), 88 deletions(-) create mode 100644 ee/app/models/embedding/application_record.rb create mode 100644 ee/db/embedding/docs/ar_internal_metadata.yml create mode 100644 ee/db/embedding/docs/schema_migrations.yml create mode 100644 ee/db/embedding/migrate/.gitkeep create mode 100644 ee/db/embedding/migrate/20230419232013_create_vector_extension.rb create mode 100644 ee/db/embedding/post_migrate/.gitkeep create mode 100644 ee/db/embedding/schema_migrations/.gitkeep create mode 100644 ee/db/embedding/schema_migrations/20230419232013 create mode 100644 ee/db/embedding/seeds.rb create mode 100644 ee/db/embedding/structure.sql rename ee/db/{ => geo}/docs/ar_internal_metadata.yml (100%) rename ee/db/{ => geo}/docs/ci_secure_file_registry.yml (100%) rename ee/db/{ => geo}/docs/container_repository_registry.yml (100%) rename ee/db/{ => geo}/docs/dependency_proxy_blob_registry.yml (100%) rename ee/db/{ => geo}/docs/dependency_proxy_manifest_registry.yml (100%) rename ee/db/{ => geo}/docs/design_registry.yml (100%) rename ee/db/{ => geo}/docs/event_log_states.yml (100%) rename ee/db/{ => geo}/docs/file_registry.yml (100%) rename ee/db/{ => geo}/docs/group_wiki_repository_registry.yml (100%) rename ee/db/{ => geo}/docs/job_artifact_registry.yml (100%) rename ee/db/{ => geo}/docs/lfs_object_registry.yml (100%) rename ee/db/{ => geo}/docs/merge_request_diff_registry.yml (100%) rename ee/db/{ => geo}/docs/package_file_registry.yml (100%) rename ee/db/{ => geo}/docs/pages_deployment_registry.yml (100%) rename ee/db/{ => geo}/docs/pipeline_artifact_registry.yml (100%) rename ee/db/{ => geo}/docs/project_registry.yml (100%) rename ee/db/{ => geo}/docs/project_wiki_repository_registry.yml (100%) rename ee/db/{ => geo}/docs/schema_migrations.yml (59%) rename ee/db/{ => geo}/docs/secondary_usage_data.yml (100%) rename ee/db/{ => geo}/docs/snippet_repository_registry.yml (100%) rename ee/db/{ => geo}/docs/terraform_state_version_registry.yml (100%) rename ee/lib/gitlab/patch/{geo_database_tasks.rb => additional_database_tasks.rb} (81%) create mode 100644 ee/lib/tasks/embedding.rake rename ee/spec/lib/gitlab/patch/{geo_database_tasks_spec.rb => additional_database_tasks_spec.rb} (55%) create mode 100644 ee/spec/models/embedding/application_record_spec.rb create mode 100644 ee/spec/models/embedding/schema_migration_spec.rb diff --git a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md index 60b091b04a27d2..c528d38ae5d76a 100644 --- a/.gitlab/issue_templates/Geo Replicate a new Git repository type.md +++ b/.gitlab/issue_templates/Geo Replicate a new Git repository type.md @@ -100,7 +100,7 @@ Geo secondary sites have a [Geo tracking database](https://gitlab.com/gitlab-org - [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). -- [ ] Add the new table to the [database dictionary](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/database/database_dictionary.md) defined in [`ee/db/docs/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/db/docs): +- [ ] Add the new table to the [database dictionary](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/database/database_dictionary.md) defined in [`ee/db/geo/docs/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/db/geo/docs): ```yaml table_name: cool_widget_registry diff --git a/.gitlab/issue_templates/Geo Replicate a new blob type.md b/.gitlab/issue_templates/Geo Replicate a new blob type.md index df2d1871a8562a..70d1dfb1105100 100644 --- a/.gitlab/issue_templates/Geo Replicate a new blob type.md +++ b/.gitlab/issue_templates/Geo Replicate a new blob type.md @@ -100,7 +100,7 @@ Geo secondary sites have a [Geo tracking database](https://gitlab.com/gitlab-org - [ ] If deviating from the above example, then be sure to order columns according to [our guidelines](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/ordering_table_columns.md). -- [ ] Add the new table to the [database dictionary](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/database/database_dictionary.md) defined in [`ee/db/docs/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/db/docs): +- [ ] Add the new table to the [database dictionary](https://gitlab.com/gitlab-org/gitlab/-/blob/master/doc/development/database/database_dictionary.md) defined in [`ee/db/geo/docs/`](https://gitlab.com/gitlab-org/gitlab/-/tree/master/ee/db/geo/docs): ```yaml table_name: cool_widget_registry diff --git a/.rubocop.yml b/.rubocop.yml index dfb2eb84fda893..e2d37ff8b88c2f 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -112,6 +112,7 @@ Lint/LastKeywordArgument: Lint/EmptyFile: Exclude: - 'db/seeds.rb' + - 'ee/db/embedding/seeds.rb' - 'ee/db/geo/seeds.rb' # This cop checks whether some constant value isn't a diff --git a/config/initializers/active_record_database_tasks.rb b/config/initializers/active_record_database_tasks.rb index 8b231f4af14239..4c08f0bbe1f74b 100644 --- a/config/initializers/active_record_database_tasks.rb +++ b/config/initializers/active_record_database_tasks.rb @@ -3,5 +3,5 @@ return unless Gitlab.ee? ActiveSupport.on_load(:active_record) do - Gitlab::Patch::GeoDatabaseTasks.patch! + Gitlab::Patch::AdditionalDatabaseTasks.patch! end diff --git a/db/docs/schema_migrations.yml b/db/docs/schema_migrations.yml index f5b52bc4db2f87..3aba0ea1d1b718 100644 --- a/db/docs/schema_migrations.yml +++ b/db/docs/schema_migrations.yml @@ -2,11 +2,12 @@ table_name: schema_migrations classes: - ActiveRecord::SchemaMigration +- Embedding::SchemaMigration - Geo::TrackingBase::SchemaMigration feature_categories: - database -description: >- - An internal table used by ActiveRecord to keep track of which migrations have been applied to the database. +description: An internal table used by ActiveRecord to keep track of which migrations + have been applied to the database. introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/commit/9ba1224867665844b117fa037e1465bb706b3685 milestone: '0.8' gitlab_schema: gitlab_internal diff --git a/doc/development/database/database_dictionary.md b/doc/development/database/database_dictionary.md index 515bf00c6f0aeb..84b76ddc34c16e 100644 --- a/doc/development/database/database_dictionary.md +++ b/doc/development/database/database_dictionary.md @@ -12,7 +12,8 @@ locate the feature categories responsible for specific database tables. ## Location Database dictionary metadata files are stored in the `gitlab` project under `db/docs/` for the `main` and `ci` databases. -For the `geo` database, the dictionary files are stored under `ee/db/docs/`. +For the `embedding` database, the dictionary files are stored under `ee/db/embedding/docs/`. +For the `geo` database, the dictionary files are stored under `ee/db/geo/docs/`. ## Example dictionary file @@ -51,7 +52,8 @@ When adding a table, you should: - `gitlab_main` table: `db/docs/` - `gitlab_ci` table: `db/docs/` - `gitlab_shared` table: `db/docs/` - - `gitlab_geo` table: `ee/db/docs/` + - `gitlab_embedding` table: `ee/db/embedding/docs/` + - `gitlab_geo` table: `ee/db/geo/docs/` 1. Name the file `.yml`, and include as much information as you know about the table. 1. Include this file in the commit with the migration that creates the table. @@ -79,7 +81,8 @@ When dropping a table, you should: - `gitlab_main` table: `db/docs/deleted_tables/` - `gitlab_ci` table: `db/docs/deleted_tables/` - `gitlab_shared` table: `db/docs/deleted_tables/` - - `gitlab_geo` table: `ee/db/docs/deleted_tables/` + - `gitlab_embedding` table: `ee/db/embedding/docs/deleted_tables/` + - `gitlab_geo` table: `ee/db/geo/docs/deleted_tables/` 1. Add the fields `removed_by_url` and `removed_in_milestone` to the dictionary file. 1. Include this change in the commit with the migration that drops the table. @@ -105,7 +108,8 @@ When adding a new view, you should: - `gitlab_main` view: `db/docs/views/` - `gitlab_ci` view: `db/docs/views/` - `gitlab_shared` view: `db/docs/views/` - - `gitlab_geo` view: `ee/db/docs/views/` + - `gitlab_embedding` view: `ee/db/embedding/docs/views/` + - `gitlab_geo` view: `ee/db/geo/docs/views/` 1. Name the file `.yml`, and include as much information as you know about the view. 1. Include this file in the commit with the migration that creates the view. @@ -133,6 +137,7 @@ When dropping a view, you should: - `gitlab_main` view: `db/docs/deleted_views/` - `gitlab_ci` view: `db/docs/deleted_views/` - `gitlab_shared` view: `db/docs/deleted_views/` - - `gitlab_geo` view: `ee/db/docs/deleted_views/` + - `gitlab_embedding` view: `ee/db/embedding/docs/deleted_views/` + - `gitlab_geo` view: `ee/db/geo/docs/deleted_views/` 1. Add the fields `removed_by_url` and `removed_in_milestone` to the dictionary file. 1. Include this change in the commit with the migration that drops the view. diff --git a/ee/app/models/embedding/application_record.rb b/ee/app/models/embedding/application_record.rb new file mode 100644 index 00000000000000..b91b792a9ea1f3 --- /dev/null +++ b/ee/app/models/embedding/application_record.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +module Embedding + class ApplicationRecord < ::ApplicationRecord + self.abstract_class = true + + connects_to database: { writing: :embedding, reading: :embedding } if Gitlab::Database.has_config?(:embedding) + + def self.model_name + @model_name ||= ActiveModel::Name.new(self, nil, name.demodulize) + end + end + + class SchemaMigration < ApplicationRecord + class << self + def all_versions + order(:version).pluck(:version) + end + end + end +end diff --git a/ee/db/embedding/docs/ar_internal_metadata.yml b/ee/db/embedding/docs/ar_internal_metadata.yml new file mode 100644 index 00000000000000..498ff1939d2055 --- /dev/null +++ b/ee/db/embedding/docs/ar_internal_metadata.yml @@ -0,0 +1,11 @@ +--- +table_name: ar_internal_metadata +classes: +- ActiveRecord::InternalMetadata +feature_categories: +- database +description: An internal table used by ActiveRecord to store information about how + the database was migrated. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118156 +milestone: '16.0' +gitlab_schema: gitlab_internal diff --git a/ee/db/embedding/docs/schema_migrations.yml b/ee/db/embedding/docs/schema_migrations.yml new file mode 100644 index 00000000000000..f04651de56c28a --- /dev/null +++ b/ee/db/embedding/docs/schema_migrations.yml @@ -0,0 +1,13 @@ +--- +table_name: schema_migrations +classes: +- ActiveRecord::SchemaMigration +- Embedding::SchemaMigration +- Geo::TrackingBase::SchemaMigration +feature_categories: +- database +description: An internal table used by ActiveRecord to keep track of which migrations + have been applied to the database. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/118156 +milestone: '16.0' +gitlab_schema: gitlab_internal diff --git a/ee/db/embedding/migrate/.gitkeep b/ee/db/embedding/migrate/.gitkeep new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/ee/db/embedding/migrate/20230419232013_create_vector_extension.rb b/ee/db/embedding/migrate/20230419232013_create_vector_extension.rb new file mode 100644 index 00000000000000..bf967cfbab5a77 --- /dev/null +++ b/ee/db/embedding/migrate/20230419232013_create_vector_extension.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class CreateVectorExtension < Gitlab::Database::Migration[2.1] + EXTENSION_NAME = 'vector' + + def up + create_extension EXTENSION_NAME + end + + def down + drop_extension EXTENSION_NAME + end +end diff --git a/ee/db/embedding/post_migrate/.gitkeep b/ee/db/embedding/post_migrate/.gitkeep new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/ee/db/embedding/schema_migrations/.gitkeep b/ee/db/embedding/schema_migrations/.gitkeep new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/ee/db/embedding/schema_migrations/20230419232013 b/ee/db/embedding/schema_migrations/20230419232013 new file mode 100644 index 00000000000000..322611b50f645f --- /dev/null +++ b/ee/db/embedding/schema_migrations/20230419232013 @@ -0,0 +1 @@ +6d926f984bdb06ce627005ec692d1348401f8a152064dddc76145ed1eab6c178 \ No newline at end of file diff --git a/ee/db/embedding/seeds.rb b/ee/db/embedding/seeds.rb new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/ee/db/embedding/structure.sql b/ee/db/embedding/structure.sql new file mode 100644 index 00000000000000..a115fbaf269a91 --- /dev/null +++ b/ee/db/embedding/structure.sql @@ -0,0 +1,18 @@ +CREATE EXTENSION IF NOT EXISTS vector; + +CREATE TABLE ar_internal_metadata ( + key character varying NOT NULL, + value character varying, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + +CREATE TABLE schema_migrations ( + version character varying NOT NULL +); + +ALTER TABLE ONLY ar_internal_metadata + ADD CONSTRAINT ar_internal_metadata_pkey PRIMARY KEY (key); + +ALTER TABLE ONLY schema_migrations + ADD CONSTRAINT schema_migrations_pkey PRIMARY KEY (version); diff --git a/ee/db/docs/ar_internal_metadata.yml b/ee/db/geo/docs/ar_internal_metadata.yml similarity index 100% rename from ee/db/docs/ar_internal_metadata.yml rename to ee/db/geo/docs/ar_internal_metadata.yml diff --git a/ee/db/docs/ci_secure_file_registry.yml b/ee/db/geo/docs/ci_secure_file_registry.yml similarity index 100% rename from ee/db/docs/ci_secure_file_registry.yml rename to ee/db/geo/docs/ci_secure_file_registry.yml diff --git a/ee/db/docs/container_repository_registry.yml b/ee/db/geo/docs/container_repository_registry.yml similarity index 100% rename from ee/db/docs/container_repository_registry.yml rename to ee/db/geo/docs/container_repository_registry.yml diff --git a/ee/db/docs/dependency_proxy_blob_registry.yml b/ee/db/geo/docs/dependency_proxy_blob_registry.yml similarity index 100% rename from ee/db/docs/dependency_proxy_blob_registry.yml rename to ee/db/geo/docs/dependency_proxy_blob_registry.yml diff --git a/ee/db/docs/dependency_proxy_manifest_registry.yml b/ee/db/geo/docs/dependency_proxy_manifest_registry.yml similarity index 100% rename from ee/db/docs/dependency_proxy_manifest_registry.yml rename to ee/db/geo/docs/dependency_proxy_manifest_registry.yml diff --git a/ee/db/docs/design_registry.yml b/ee/db/geo/docs/design_registry.yml similarity index 100% rename from ee/db/docs/design_registry.yml rename to ee/db/geo/docs/design_registry.yml diff --git a/ee/db/docs/event_log_states.yml b/ee/db/geo/docs/event_log_states.yml similarity index 100% rename from ee/db/docs/event_log_states.yml rename to ee/db/geo/docs/event_log_states.yml diff --git a/ee/db/docs/file_registry.yml b/ee/db/geo/docs/file_registry.yml similarity index 100% rename from ee/db/docs/file_registry.yml rename to ee/db/geo/docs/file_registry.yml diff --git a/ee/db/docs/group_wiki_repository_registry.yml b/ee/db/geo/docs/group_wiki_repository_registry.yml similarity index 100% rename from ee/db/docs/group_wiki_repository_registry.yml rename to ee/db/geo/docs/group_wiki_repository_registry.yml diff --git a/ee/db/docs/job_artifact_registry.yml b/ee/db/geo/docs/job_artifact_registry.yml similarity index 100% rename from ee/db/docs/job_artifact_registry.yml rename to ee/db/geo/docs/job_artifact_registry.yml diff --git a/ee/db/docs/lfs_object_registry.yml b/ee/db/geo/docs/lfs_object_registry.yml similarity index 100% rename from ee/db/docs/lfs_object_registry.yml rename to ee/db/geo/docs/lfs_object_registry.yml diff --git a/ee/db/docs/merge_request_diff_registry.yml b/ee/db/geo/docs/merge_request_diff_registry.yml similarity index 100% rename from ee/db/docs/merge_request_diff_registry.yml rename to ee/db/geo/docs/merge_request_diff_registry.yml diff --git a/ee/db/docs/package_file_registry.yml b/ee/db/geo/docs/package_file_registry.yml similarity index 100% rename from ee/db/docs/package_file_registry.yml rename to ee/db/geo/docs/package_file_registry.yml diff --git a/ee/db/docs/pages_deployment_registry.yml b/ee/db/geo/docs/pages_deployment_registry.yml similarity index 100% rename from ee/db/docs/pages_deployment_registry.yml rename to ee/db/geo/docs/pages_deployment_registry.yml diff --git a/ee/db/docs/pipeline_artifact_registry.yml b/ee/db/geo/docs/pipeline_artifact_registry.yml similarity index 100% rename from ee/db/docs/pipeline_artifact_registry.yml rename to ee/db/geo/docs/pipeline_artifact_registry.yml diff --git a/ee/db/docs/project_registry.yml b/ee/db/geo/docs/project_registry.yml similarity index 100% rename from ee/db/docs/project_registry.yml rename to ee/db/geo/docs/project_registry.yml diff --git a/ee/db/docs/project_wiki_repository_registry.yml b/ee/db/geo/docs/project_wiki_repository_registry.yml similarity index 100% rename from ee/db/docs/project_wiki_repository_registry.yml rename to ee/db/geo/docs/project_wiki_repository_registry.yml diff --git a/ee/db/docs/schema_migrations.yml b/ee/db/geo/docs/schema_migrations.yml similarity index 59% rename from ee/db/docs/schema_migrations.yml rename to ee/db/geo/docs/schema_migrations.yml index 695e0fb698bdc3..7a74cc4ccbb8ad 100644 --- a/ee/db/docs/schema_migrations.yml +++ b/ee/db/geo/docs/schema_migrations.yml @@ -1,12 +1,13 @@ --- table_name: schema_migrations -description: >- - An internal table used by ActiveRecord to keep track of which migrations have been applied to the database. +description: An internal table used by ActiveRecord to keep track of which migrations + have been applied to the database. introduced_by_url: TODO milestone: TODO feature_categories: - geo_replication classes: -- Geo::TrackingBase::SchemaMigration - ActiveRecord::SchemaMigration +- Embedding::SchemaMigration +- Geo::TrackingBase::SchemaMigration gitlab_schema: gitlab_internal diff --git a/ee/db/docs/secondary_usage_data.yml b/ee/db/geo/docs/secondary_usage_data.yml similarity index 100% rename from ee/db/docs/secondary_usage_data.yml rename to ee/db/geo/docs/secondary_usage_data.yml diff --git a/ee/db/docs/snippet_repository_registry.yml b/ee/db/geo/docs/snippet_repository_registry.yml similarity index 100% rename from ee/db/docs/snippet_repository_registry.yml rename to ee/db/geo/docs/snippet_repository_registry.yml diff --git a/ee/db/docs/terraform_state_version_registry.yml b/ee/db/geo/docs/terraform_state_version_registry.yml similarity index 100% rename from ee/db/docs/terraform_state_version_registry.yml rename to ee/db/geo/docs/terraform_state_version_registry.yml diff --git a/ee/lib/ee/gitlab/database.rb b/ee/lib/ee/gitlab/database.rb index 3812f3b09e7454..06cc675b2f7270 100644 --- a/ee/lib/ee/gitlab/database.rb +++ b/ee/lib/ee/gitlab/database.rb @@ -5,16 +5,23 @@ module Gitlab module Database extend ActiveSupport::Concern + EMBEDDING_DATABASE_NAME = 'embedding' + EMBEDDING_DATABASE_DIR = 'ee/db/embedding' + GEO_DATABASE_NAME = 'geo' GEO_DATABASE_DIR = 'ee/db/geo' - EE_DATABASE_NAMES = [GEO_DATABASE_NAME].freeze + + EE_DATABASES_NAME_TO_DIR = { + EMBEDDING_DATABASE_NAME => EMBEDDING_DATABASE_DIR, + GEO_DATABASE_NAME => GEO_DATABASE_DIR + }.freeze class_methods do extend ::Gitlab::Utils::Override override :all_database_names def all_database_names - super + EE_DATABASE_NAMES + super + EE_DATABASES_NAME_TO_DIR.keys end override :check_postgres_version_and_print_warning @@ -27,21 +34,26 @@ def check_postgres_version_and_print_warning override :database_base_models def database_base_models @database_base_models_ee ||= super.merge( + embedding: ::Embedding::ApplicationRecord.connection_class? ? ::Embedding::ApplicationRecord : nil, geo: ::Geo::TrackingBase.connection_class? ? ::Geo::TrackingBase : nil ).compact.with_indifferent_access.freeze end + override :database_base_models_using_load_balancing + def database_base_models_using_load_balancing + @database_base_models_using_load_balancing ||= super.merge( + embedding: ::Embedding::ApplicationRecord.connection_class? ? ::Embedding::ApplicationRecord : nil + ).compact.with_indifferent_access.freeze + end + override :schemas_to_base_models def schemas_to_base_models @schemas_to_base_models_ee ||= super.merge( + gitlab_embedding: [self.database_base_models[:embedding]].compact, gitlab_geo: [self.database_base_models[:geo]].compact ).compact.with_indifferent_access.freeze end - def geo_database?(name) - name.to_s == GEO_DATABASE_NAME - end - def geo_db_config_with_default_pool_size db_config_object = Geo::TrackingBase.connection_db_config diff --git a/ee/lib/ee/gitlab/database/gitlab_schema.rb b/ee/lib/ee/gitlab/database/gitlab_schema.rb index 9de3b60081f9e2..db1a27cc62078d 100644 --- a/ee/lib/ee/gitlab/database/gitlab_schema.rb +++ b/ee/lib/ee/gitlab/database/gitlab_schema.rb @@ -6,29 +6,35 @@ module Database module GitlabSchema extend ActiveSupport::Concern - EE_DICTIONARY_PATH = 'ee/db/docs' - class_methods do extend ::Gitlab::Utils::Override override :dictionary_path_globs def dictionary_path_globs - super + [Rails.root.join(EE_DICTIONARY_PATH, '*.yml')] + super + Gitlab::Database::EE_DATABASES_NAME_TO_DIR.map do |_, ee_db_dir| + Rails.root.join(ee_db_dir, 'docs', '*.yml') + end end override :view_path_globs def view_path_globs - super + [Rails.root.join(EE_DICTIONARY_PATH, 'views', '*.yml')] + super + Gitlab::Database::EE_DATABASES_NAME_TO_DIR.map do |_, ee_db_dir| + Rails.root.join(ee_db_dir, 'docs', 'views', '*.yml') + end end override :deleted_tables_path_globs def deleted_tables_path_globs - super + [Rails.root.join(EE_DICTIONARY_PATH, 'deleted_tables', '*.yml')] + super + Gitlab::Database::EE_DATABASES_NAME_TO_DIR.map do |_, ee_db_dir| + Rails.root.join(ee_db_dir, 'docs', 'deleted_tables', '*.yml') + end end override :deleted_views_path_globs def deleted_views_path_globs - super + [Rails.root.join(EE_DICTIONARY_PATH, 'deleted_views', '*.yml')] + super + Gitlab::Database::EE_DATABASES_NAME_TO_DIR.map do |_, ee_db_dir| + Rails.root.join(ee_db_dir, 'docs', 'deleted_views', '*.yml') + end end end end diff --git a/ee/lib/gitlab/patch/geo_database_tasks.rb b/ee/lib/gitlab/patch/additional_database_tasks.rb similarity index 81% rename from ee/lib/gitlab/patch/geo_database_tasks.rb rename to ee/lib/gitlab/patch/additional_database_tasks.rb index 444167e0f347c8..6107880c609ed3 100644 --- a/ee/lib/gitlab/patch/geo_database_tasks.rb +++ b/ee/lib/gitlab/patch/additional_database_tasks.rb @@ -5,20 +5,20 @@ module Gitlab module Patch - module GeoDatabaseTasks + module AdditionalDatabaseTasks # Returns the proper path for the structure.sql and schema_cache.yml - # files for the Geo tracking database. + # files for additional databases. module ActiveRecordDatabaseTasksDumpFilename def dump_filename(db_config_name, format = ApplicationRecord.schema_format) - return super unless Gitlab::Database.geo_database?(db_config_name) + return super unless Gitlab::Database::EE_DATABASES_NAME_TO_DIR.key?(db_config_name.to_s) - Rails.root.join(Gitlab::Database::GEO_DATABASE_DIR, 'structure.sql').to_s + Rails.root.join(Gitlab::Database::EE_DATABASES_NAME_TO_DIR[db_config_name.to_s], 'structure.sql').to_s end def cache_dump_filename(db_config_name, schema_cache_path: nil) - return super unless Gitlab::Database.geo_database?(db_config_name) + return super unless Gitlab::Database::EE_DATABASES_NAME_TO_DIR.key?(db_config_name.to_s) - Rails.root.join(Gitlab::Database::GEO_DATABASE_DIR, 'schema_cache.yml').to_s + Rails.root.join(Gitlab::Database::EE_DATABASES_NAME_TO_DIR[db_config_name.to_s], 'schema_cache.yml').to_s end end diff --git a/ee/lib/tasks/embedding.rake b/ee/lib/tasks/embedding.rake new file mode 100644 index 00000000000000..1444f4ee442818 --- /dev/null +++ b/ee/lib/tasks/embedding.rake @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +task spec: ['db:test:prepare:embedding'] + +db_namespace = namespace :db do + namespace :seed do + seed_loader = Class.new do + def self.load_seed + load('ee/db/embedding/seeds.rb') + end + end + + desc "Loads the seed data from ee/db/embedding/seeds.rb" + task embedding: :load_config do + db_namespace["abort_if_pending_migrations:embedding"].invoke + ActiveRecord::Tasks::DatabaseTasks.seed_loader = seed_loader + ActiveRecord::Tasks::DatabaseTasks.load_seed + end + end +end diff --git a/ee/spec/lib/ee/gitlab/database/docs/docs_spec.rb b/ee/spec/lib/ee/gitlab/database/docs/docs_spec.rb index e6d3457922798d..cf4aed97c28da2 100644 --- a/ee/spec/lib/ee/gitlab/database/docs/docs_spec.rb +++ b/ee/spec/lib/ee/gitlab/database/docs/docs_spec.rb @@ -122,20 +122,42 @@ def multiline_error(title, lines) # rubocop:enable Naming/HeredocDelimiterNaming end -RSpec.describe 'Views documentation', feature_category: :database do - database_base_models = Gitlab::Database.database_base_models.select { |k, _| k == 'geo' } - views = database_base_models.flat_map { |_, m| m.connection.views }.sort.uniq - directory_path = File.join('ee', 'db', 'docs', 'views') - required_fields = %i[feature_categories view_name gitlab_schema] +RSpec.describe 'embedding database documentation', feature_category: :database do + context 'for views' do + database_base_models = Gitlab::Database.database_base_models.select { |k, _| k == 'embedding' } + views = database_base_models.flat_map { |_, m| m.connection.views }.sort.uniq + directory_path = File.join('ee', 'db', 'embedding', 'docs', 'views') + required_fields = %i[feature_categories view_name gitlab_schema] + + include_examples 'validate dictionary', views, directory_path, required_fields + end + + context 'for tables' do + database_base_models = Gitlab::Database.database_base_models.select { |k, _| k == 'embedding' } + tables = database_base_models.flat_map { |_, m| m.connection.tables }.sort.uniq + directory_path = File.join('ee', 'db', 'embedding', 'docs') + required_fields = %i[feature_categories table_name gitlab_schema] - include_examples 'validate dictionary', views, directory_path, required_fields + include_examples 'validate dictionary', tables, directory_path, required_fields + end end -RSpec.describe 'Tables documentation', feature_category: :database do - database_base_models = Gitlab::Database.database_base_models.select { |k, _| k == 'geo' } - tables = database_base_models.flat_map { |_, m| m.connection.tables }.sort.uniq - directory_path = File.join('ee', 'db', 'docs') - required_fields = %i[feature_categories table_name gitlab_schema] +RSpec.describe 'geo database documentation', feature_category: :database do + context 'for views' do + database_base_models = Gitlab::Database.database_base_models.select { |k, _| k == 'geo' } + views = database_base_models.flat_map { |_, m| m.connection.views }.sort.uniq + directory_path = File.join('ee', 'db', 'geo', 'docs', 'views') + required_fields = %i[feature_categories view_name gitlab_schema] - include_examples 'validate dictionary', tables, directory_path, required_fields + include_examples 'validate dictionary', views, directory_path, required_fields + end + + context 'for tables' do + database_base_models = Gitlab::Database.database_base_models.select { |k, _| k == 'geo' } + tables = database_base_models.flat_map { |_, m| m.connection.tables }.sort.uniq + directory_path = File.join('ee', 'db', 'geo', 'docs') + required_fields = %i[feature_categories table_name gitlab_schema] + + include_examples 'validate dictionary', tables, directory_path, required_fields + end end diff --git a/ee/spec/lib/ee/gitlab/database/gitlab_schema_spec.rb b/ee/spec/lib/ee/gitlab/database/gitlab_schema_spec.rb index 274344022eb627..58de9d260bc2e8 100644 --- a/ee/spec/lib/ee/gitlab/database/gitlab_schema_spec.rb +++ b/ee/spec/lib/ee/gitlab/database/gitlab_schema_spec.rb @@ -12,7 +12,10 @@ describe '.views_and_tables_to_schema' do it 'all tables and views are unique' do table_names = load_schemas(described_class::DICTIONARY_PATH).keys - ee_table_names = load_schemas(described_class::EE_DICTIONARY_PATH).keys + ee_table_names = Gitlab::Database::EE_DATABASES_NAME_TO_DIR + .flat_map do |_, ee_db_dir| + load_schemas(ee_db_dir).keys + end duplicated_tables = table_names & ee_table_names expect(duplicated_tables).to be_empty, \ diff --git a/ee/spec/lib/gitlab/patch/geo_database_tasks_spec.rb b/ee/spec/lib/gitlab/patch/additional_database_tasks_spec.rb similarity index 55% rename from ee/spec/lib/gitlab/patch/geo_database_tasks_spec.rb rename to ee/spec/lib/gitlab/patch/additional_database_tasks_spec.rb index bebaf915e70379..41f9a54f2e0659 100644 --- a/ee/spec/lib/gitlab/patch/geo_database_tasks_spec.rb +++ b/ee/spec/lib/gitlab/patch/additional_database_tasks_spec.rb @@ -2,57 +2,61 @@ require 'spec_helper' -RSpec.describe Gitlab::Patch::GeoDatabaseTasks, feature_category: :geo_replication do - describe Gitlab::Patch::GeoDatabaseTasks::ActiveRecordDatabaseTasksDumpFilename do +RSpec.describe Gitlab::Patch::AdditionalDatabaseTasks, feature_category: :geo_replication do + describe Gitlab::Patch::AdditionalDatabaseTasks::ActiveRecordDatabaseTasksDumpFilename do subject do Class.new do - prepend Gitlab::Patch::GeoDatabaseTasks::ActiveRecordDatabaseTasksDumpFilename + prepend Gitlab::Patch::AdditionalDatabaseTasks::ActiveRecordDatabaseTasksDumpFilename - def dump_filename(db_config_name, format = ApplicationRecord.schema_format) - 'foo.sql' + def dump_filename(*) + Rails.root.join('foo.sql').to_s end - def cache_dump_filename(db_config_name, format = ApplicationRecord.schema_format) - 'bar.yml' + def cache_dump_filename(*) + Rails.root.join('bar.yml').to_s end end.new end describe '#dump_filename' do - context 'with geo database config name' do - it 'returns the path for the structure.sql file in the Geo database dir' do - expect(subject.dump_filename(:geo)).to eq Rails.root.join('ee/db/geo/structure.sql').to_s - end + using RSpec::Parameterized::TableSyntax + + where(:db_config_name, :structure_path) do + :main | 'foo.sql' + :embedding | 'ee/db/embedding/structure.sql' + :geo | 'ee/db/geo/structure.sql' end - context 'with other database config name' do - it 'calls super' do - expect(subject.dump_filename(:main)).to eq 'foo.sql' + with_them do + it 'returns the correct path for the structure.sql file' do + expect(subject.dump_filename(db_config_name)).to eq Rails.root.join(structure_path).to_s end end end describe '#cache_dump_filename' do - context 'with geo database config name' do - it 'returns the path for the schema_cache file in the Geo database dir' do - expect(subject.cache_dump_filename(:geo)).to eq Rails.root.join('ee/db/geo/schema_cache.yml').to_s - end + using RSpec::Parameterized::TableSyntax + + where(:db_config_name, :schema_cache_path) do + :main | 'bar.yml' + :embedding | 'ee/db/embedding/schema_cache.yml' + :geo | 'ee/db/geo/schema_cache.yml' end - context 'with other database config name' do - it 'calls super' do - expect(subject.cache_dump_filename(:main)).to eq 'bar.yml' + with_them do + it 'returns the path for the schema_cache file in the Geo database dir' do + expect(subject.cache_dump_filename(db_config_name)).to eq Rails.root.join(schema_cache_path).to_s end end end end - describe Gitlab::Patch::GeoDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath do + describe Gitlab::Patch::AdditionalDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath do describe '#configured_migrate_path' do context 'when super returns nil' do subject do Class.new do - prepend Gitlab::Patch::GeoDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath + prepend Gitlab::Patch::AdditionalDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath def configured_migrate_path nil @@ -68,7 +72,7 @@ def configured_migrate_path context 'when super returns only one regular migration path' do subject do Class.new do - prepend Gitlab::Patch::GeoDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath + prepend Gitlab::Patch::AdditionalDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath def configured_migrate_path 'ee/db/geo/migrate' @@ -84,7 +88,7 @@ def configured_migrate_path context 'when super returns only one post migrations path' do subject do Class.new do - prepend Gitlab::Patch::GeoDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath + prepend Gitlab::Patch::AdditionalDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath def configured_migrate_path 'ee/db/geo/post_migrate' @@ -100,7 +104,7 @@ def configured_migrate_path context 'when super does not include a post migrations path' do subject do Class.new do - prepend Gitlab::Patch::GeoDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath + prepend Gitlab::Patch::AdditionalDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath def configured_migrate_path 'ee/db/geo/migrate' @@ -116,7 +120,7 @@ def configured_migrate_path context 'when super includes a post migrations path' do subject do Class.new do - prepend Gitlab::Patch::GeoDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath + prepend Gitlab::Patch::AdditionalDatabaseTasks::ActiveRecordMigrationConfiguredMigratePath def configured_migrate_path ['ee/db/geo/migrate', 'ee/db/geo/post_migrate'] diff --git a/ee/spec/models/embedding/application_record_spec.rb b/ee/spec/models/embedding/application_record_spec.rb new file mode 100644 index 00000000000000..3dc0e2c00b3a68 --- /dev/null +++ b/ee/spec/models/embedding/application_record_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Embedding::ApplicationRecord, feature_category: :database do + describe '.model_name' do + subject { described_class.model_name } + + it 'removes the prefix' do + expect(subject.collection).to eq 'application_records' + end + end +end diff --git a/ee/spec/models/embedding/schema_migration_spec.rb b/ee/spec/models/embedding/schema_migration_spec.rb new file mode 100644 index 00000000000000..624a84639f5280 --- /dev/null +++ b/ee/spec/models/embedding/schema_migration_spec.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Embedding::SchemaMigration, feature_category: :database do + describe '.all_versions' do + subject { described_class.all_versions } + + it 'returns all versions' do + allow(described_class).to receive(:order).with(:version).and_return([{ version: '2' }, { version: '1' }]) + + expect(subject).to eq %w[2 1] + end + end +end diff --git a/ee/spec/support/helpers/ee/require_migration.rb b/ee/spec/support/helpers/ee/require_migration.rb index 456e3317c89d1d..98846055d32572 100644 --- a/ee/spec/support/helpers/ee/require_migration.rb +++ b/ee/spec/support/helpers/ee/require_migration.rb @@ -4,7 +4,12 @@ module EE module RequireMigration extend ActiveSupport::Concern - MIGRATION_FOLDERS = %w[ee/db/geo/migrate ee/db/geo/post_migrate].freeze + MIGRATION_FOLDERS = %w[ + ee/db/embedding/migrate + ee/db/embedding/post_migrate + ee/db/geo/migrate + ee/db/geo/post_migrate + ].freeze class_methods do def migration_folders diff --git a/lib/gitlab/database/gitlab_schema.rb b/lib/gitlab/database/gitlab_schema.rb index 926a4aeedf1a97..f11458945f4134 100644 --- a/lib/gitlab/database/gitlab_schema.rb +++ b/lib/gitlab/database/gitlab_schema.rb @@ -23,6 +23,7 @@ def self.table_schemas(tables, undefined: true) tables.map { |table| table_schema(table, undefined: undefined) }.to_set end + # rubocop:disable Metrics/CyclomaticComplexity def self.table_schema(name, undefined: true) schema_name, table_name = name.split('.', 2) # Strip schema name like: `public.` @@ -57,6 +58,8 @@ def self.table_schema(name, undefined: true) return :gitlab_ci if table_name.start_with?('_test_gitlab_ci_') + return :gitlab_embedding if table_name.start_with?('_test_gitlab_embedding_') + return :gitlab_geo if table_name.start_with?('_test_gitlab_geo_') # All tables that start with `_test_` without a following schema are shared and ignored @@ -68,6 +71,7 @@ def self.table_schema(name, undefined: true) # When undefined it's best to return a unique name so that we don't incorrectly assume that 2 undefined schemas belong on the same database undefined ? :"undefined_#{table_name}" : nil end + # rubocop:enable Metrics/CyclomaticComplexity def self.dictionary_path_globs [Rails.root.join(DICTIONARY_PATH, '*.yml')] diff --git a/lib/gitlab/database/tables_locker.rb b/lib/gitlab/database/tables_locker.rb index 1b6ab3fb24bf30..0b0d46f4b0ed7a 100644 --- a/lib/gitlab/database/tables_locker.rb +++ b/lib/gitlab/database/tables_locker.rb @@ -3,7 +3,7 @@ module Gitlab module Database class TablesLocker - GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_geo].freeze + GITLAB_SCHEMAS_TO_IGNORE = %i[gitlab_embedding gitlab_geo].freeze def initialize(logger: nil, dry_run: false) @logger = logger diff --git a/lib/gitlab/patch/database_config.rb b/lib/gitlab/patch/database_config.rb index 20d8f7be8fd43c..8a7566f6e0e23e 100644 --- a/lib/gitlab/patch/database_config.rb +++ b/lib/gitlab/patch/database_config.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true # The purpose of this code is to set the migrations path -# for the Geo tracking database. +# for the Geo tracking database and the embedding database. module Gitlab module Patch module DatabaseConfig @@ -10,13 +10,17 @@ module DatabaseConfig def database_configuration super.to_h do |env, configs| if Gitlab.ee? - if configs.key?("geo") - migrations_paths = Array(configs["geo"]["migrations_paths"]) - migrations_paths << "ee/db/geo/migrate" if migrations_paths.empty? - migrations_paths << "ee/db/geo/post_migrate" unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] + ee_databases = %w[embedding geo] - configs["geo"]["migrations_paths"] = migrations_paths.uniq - configs["geo"]["schema_migrations_path"] = "ee/db/geo/schema_migrations" if configs["geo"]["schema_migrations_path"].blank? + ee_databases.each do |ee_db_name| + next unless configs.key?(ee_db_name) + + migrations_paths = Array(configs[ee_db_name]['migrations_paths']) + migrations_paths << File.join('ee', 'db', ee_db_name, 'migrate') if migrations_paths.empty? + migrations_paths << File.join('ee', 'db', ee_db_name, 'post_migrate') unless ENV['SKIP_POST_DEPLOYMENT_MIGRATIONS'] + + configs[ee_db_name]['migrations_paths'] = migrations_paths.uniq + configs[ee_db_name]['schema_migrations_path'] = File.join('ee', 'db', ee_db_name, 'schema_migrations') if configs[ee_db_name]['schema_migrations_path'].blank? end end diff --git a/lib/tasks/gitlab/db.rake b/lib/tasks/gitlab/db.rake index 963fe23c682d4c..7d95f744d44982 100644 --- a/lib/tasks/gitlab/db.rake +++ b/lib/tasks/gitlab/db.rake @@ -4,6 +4,7 @@ databases = ActiveRecord::Tasks::DatabaseTasks.setup_initial_database_yaml def each_database(databases, include_geo: false) ActiveRecord::Tasks::DatabaseTasks.for_each(databases) do |database| + next if database == 'embedding' next if !include_geo && database == 'geo' yield database @@ -472,15 +473,19 @@ namespace :gitlab do end namespace :dictionary do - DB_DOCS_PATH = File.join(Rails.root, 'db', 'docs') - EE_DICTIONARY_PATH = File.join(Rails.root, 'ee', 'db', 'docs') + DB_DOCS_PATH = Rails.root.join('db', 'docs') desc 'Generate database docs yaml' task generate: :environment do next if Gitlab.jh? FileUtils.mkdir_p(DB_DOCS_PATH) - FileUtils.mkdir_p(EE_DICTIONARY_PATH) if Gitlab.ee? + + if Gitlab.ee? + Gitlab::Database::EE_DATABASES_NAME_TO_DIR.each do |_, ee_db_dir| + FileUtils.mkdir_p(Rails.root.join(ee_db_dir, 'docs')) + end + end Rails.application.eager_load! @@ -558,7 +563,11 @@ namespace :gitlab do def dictionary_file_path(source_name, views, database) sub_directory = views.include?(source_name) ? 'views' : '' - path = database == 'geo' ? EE_DICTIONARY_PATH : DB_DOCS_PATH + path = if Gitlab.ee? && Gitlab::Database::EE_DATABASES_NAME_TO_DIR.key?(database.to_s) + Rails.root.join(Gitlab::Database::EE_DATABASES_NAME_TO_DIR[database.to_s], 'docs') + else + DB_DOCS_PATH + end File.join(path, sub_directory, "#{source_name}.yml") end diff --git a/rubocop/migration_helpers.rb b/rubocop/migration_helpers.rb index db8fc079774b81..d49280c39e42f4 100644 --- a/rubocop/migration_helpers.rb +++ b/rubocop/migration_helpers.rb @@ -42,11 +42,11 @@ def in_ee_background_migration?(node) end def in_deployment_migration?(node) - dirname(node).end_with?('db/migrate', 'db/geo/migrate') + dirname(node).end_with?('db/migrate', 'db/embedding/migrate', 'db/geo/migrate') end def in_post_deployment_migration?(node) - dirname(node).end_with?('db/post_migrate', 'db/geo/post_migrate') + dirname(node).end_with?('db/post_migrate', 'db/embedding/post_migrate', 'db/geo/post_migrate') end # Returns true if we've defined an 'EnforcedSince' variable in rubocop.yml and the migration version diff --git a/spec/lib/gitlab/database/gitlab_schema_spec.rb b/spec/lib/gitlab/database/gitlab_schema_spec.rb index b187b29c270ee8..3d1e3a1c1af177 100644 --- a/spec/lib/gitlab/database/gitlab_schema_spec.rb +++ b/spec/lib/gitlab/database/gitlab_schema_spec.rb @@ -51,8 +51,10 @@ context "for #{db_config_name} using #{db_class}" do let(:db_data_sources) { db_class.connection.data_sources } - # The Geo database does not share the same structure as all decomposed databases - subject { described_class.views_and_tables_to_schema.select { |_, v| v != :gitlab_geo } } + # The embedding and Geo databases do not share the same structure as all decomposed databases + subject do + described_class.views_and_tables_to_schema.reject { |_, v| v == :gitlab_embedding || v == :gitlab_geo } + end it 'new data sources are added' do missing_data_sources = db_data_sources.to_set - subject.keys diff --git a/spec/tasks/gitlab/db_rake_spec.rb b/spec/tasks/gitlab/db_rake_spec.rb index 45d0c0509492d5..fb45bc0864dfd2 100644 --- a/spec/tasks/gitlab/db_rake_spec.rb +++ b/spec/tasks/gitlab/db_rake_spec.rb @@ -566,6 +566,7 @@ end if Gitlab.ee? + allow(File).to receive(:open).with(Rails.root.join(Gitlab::Database::EMBEDDING_DATABASE_DIR, 'structure.sql').to_s, any_args).and_yield(output) allow(File).to receive(:open).with(Rails.root.join(Gitlab::Database::GEO_DATABASE_DIR, 'structure.sql').to_s, any_args).and_yield(output) end end -- GitLab