From d35f1f683ecbdf4f256bc0903b047cad476ec9fe Mon Sep 17 00:00:00 2001 From: Serena Fang Date: Tue, 15 Mar 2022 21:27:44 -0500 Subject: [PATCH 1/8] Create NamespaceDetails table Changelog: changed --- ...20220316022505_create_namespace_details.rb | 30 +++++++++++++++++++ db/schema_migrations/20220316022505 | 1 + db/structure.sql | 23 ++++++++++++++ 3 files changed, 54 insertions(+) create mode 100644 db/migrate/20220316022505_create_namespace_details.rb create mode 100644 db/schema_migrations/20220316022505 diff --git a/db/migrate/20220316022505_create_namespace_details.rb b/db/migrate/20220316022505_create_namespace_details.rb new file mode 100644 index 00000000000000..2f3c3361493d3f --- /dev/null +++ b/db/migrate/20220316022505_create_namespace_details.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +# See https://docs.gitlab.com/ee/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateNamespaceDetails < Gitlab::Database::Migration[1.0] + # When using the methods "add_concurrent_index" or "remove_concurrent_index" + # you must disable the use of transactions + # as these methods can not run in an existing transaction. + # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure + # that either of them is the _only_ method called in the migration, + # any other changes should go in a separate migration. + # This ensures that upon failure _only_ the index creation or removing fails + # and can be retried or reverted easily. + # + # To disable transactions uncomment the following line and remove these + # comments: + # disable_ddl_transaction! + + def change + create_table :namespace_details do |t| + t.integer :namespace_id + t.string :description + t.text :description_html + t.integer :cached_markdown_version + + t.timestamps null: false + end + end +end diff --git a/db/schema_migrations/20220316022505 b/db/schema_migrations/20220316022505 new file mode 100644 index 00000000000000..dd6bed30e8a09c --- /dev/null +++ b/db/schema_migrations/20220316022505 @@ -0,0 +1 @@ +c974e1a600323bac9b913e9e382384c302037ed6d1fc1df3b747471810293167 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index b0afd780e305b0..a1785a551d482d 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -17555,6 +17555,25 @@ CREATE TABLE namespace_ci_cd_settings ( allow_stale_runner_pruning boolean DEFAULT false NOT NULL ); +CREATE TABLE namespace_details ( + id bigint NOT NULL, + namespace_id integer, + description character varying, + description_html text, + cached_markdown_version integer, + created_at timestamp(6) without time zone NOT NULL, + updated_at timestamp(6) without time zone NOT NULL +); + +CREATE SEQUENCE namespace_details_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE namespace_details_id_seq OWNED BY namespace_details.id; + CREATE TABLE namespace_limits ( additional_purchased_storage_size bigint DEFAULT 0 NOT NULL, additional_purchased_storage_ends_on date, @@ -23354,6 +23373,8 @@ ALTER TABLE ONLY namespace_admin_notes ALTER COLUMN id SET DEFAULT nextval('name ALTER TABLE ONLY namespace_bans ALTER COLUMN id SET DEFAULT nextval('namespace_bans_id_seq'::regclass); +ALTER TABLE ONLY namespace_details ALTER COLUMN id SET DEFAULT nextval('namespace_details_id_seq'::regclass); + ALTER TABLE ONLY namespace_statistics ALTER COLUMN id SET DEFAULT nextval('namespace_statistics_id_seq'::regclass); ALTER TABLE ONLY namespaces ALTER COLUMN id SET DEFAULT nextval('namespaces_id_seq'::regclass); @@ -25370,6 +25391,8 @@ ALTER TABLE ONLY namespace_bans ALTER TABLE ONLY namespace_ci_cd_settings ADD CONSTRAINT namespace_ci_cd_settings_pkey PRIMARY KEY (namespace_id); +ALTER TABLE ONLY namespace_details + ADD CONSTRAINT namespace_details_pkey PRIMARY KEY (id); ALTER TABLE ONLY namespace_limits ADD CONSTRAINT namespace_limits_pkey PRIMARY KEY (namespace_id); -- GitLab From cc2cb6d85165d8878f31222bcebf8511d8cb0523 Mon Sep 17 00:00:00 2001 From: Serena Fang Date: Tue, 15 Mar 2022 21:43:07 -0500 Subject: [PATCH 2/8] Add namespace details feature flag Prepare migrations for NamespaceDetail Todo: figure out backfilling Update create namespace detail table Add post migration file Fix rubocop errors Fix rubocop error Add backfill namespace details spec Run rails migrate Add namespace details to gitlab schemas Select other columns Remove schema version number Fix rubocop errors Rename files Add namespace scope Fix table name Fix rspec errors Remove unnecessary change Move spec file Remove extra file Run rails migrate Run rails migrate Fix db structure Rename NamespaceDetail to be in module NamespaceDetail -> Namespace::Detail Sync namespace details Remove extra file Update project namespace details when project description changes Move namespace details to new method Remove extra attributes Remove extra line Check if project namespace present Create backfill project namespace details Need migration to backfill project namespace details Backfill project namespace details spec Fix specs Fix specs Add db docs Ignore project namespaces Apply reviewer suggestions Add feature flag Remove trailing whitespace Rebase with master Loose foreign keys Update ff milestone Move backfill files Remove schema migrations Add description validation Add namespace details trigger Add database trigger testing my network Revert "testing my network" This reverts commit 81e5160cc0b2293c220bb36e381259e523d95917. Run rubocop linter Remove create trigger Add new line Apply reviewer suggestions Add insert trigger Fix structure sql Add project namespace trigger Update db structure Fix some of the failures Fix some of the failures Add cr remarks Check for project type Check namespace type Change trigger sql Update db structure Update trigger code Update db schema Do not reload project namespace Change db structure Change db structure Fix db structure Fix db structure Fix db structure Remove extra spaces --- .../style/class_and_module_children.yml | 1 + app/models/namespace.rb | 8 ++ app/models/namespace/detail.rb | 9 ++ app/models/project.rb | 8 ++ db/docs/namespace_details.yml | 9 ++ ...20220316022505_create_namespace_details.rb | 39 +++---- ...4_create_sync_namespace_details_trigger.rb | 68 ++++++++++++ ..._sync_project_namespace_details_trigger.rb | 66 ++++++++++++ db/schema_migrations/20220506154054 | 1 + db/schema_migrations/20220524184149 | 1 + db/structure.sql | 101 +++++++++++++++--- lib/gitlab/database/gitlab_schemas.yml | 1 + ...ate_sync_namespace_details_trigger_spec.rb | 76 +++++++++++++ ..._project_namespace_details_trigger_spec.rb | 73 +++++++++++++ spec/models/namespace/detail_spec.rb | 40 +++++++ spec/models/namespace_spec.rb | 1 + spec/requests/api/project_import_spec.rb | 2 +- spec/services/groups/create_service_spec.rb | 9 ++ 18 files changed, 472 insertions(+), 41 deletions(-) create mode 100644 app/models/namespace/detail.rb create mode 100644 db/docs/namespace_details.yml create mode 100644 db/migrate/20220506154054_create_sync_namespace_details_trigger.rb create mode 100644 db/migrate/20220524184149_create_sync_project_namespace_details_trigger.rb create mode 100644 db/schema_migrations/20220506154054 create mode 100644 db/schema_migrations/20220524184149 create mode 100644 spec/migrations/20220506154054_create_sync_namespace_details_trigger_spec.rb create mode 100644 spec/migrations/20220524184149_create_sync_project_namespace_details_trigger_spec.rb create mode 100644 spec/models/namespace/detail_spec.rb diff --git a/.rubocop_todo/style/class_and_module_children.yml b/.rubocop_todo/style/class_and_module_children.yml index fab05667adb44f..6d145f11579c5e 100644 --- a/.rubocop_todo/style/class_and_module_children.yml +++ b/.rubocop_todo/style/class_and_module_children.yml @@ -292,6 +292,7 @@ Style/ClassAndModuleChildren: - 'app/models/merge_request/metrics.rb' - 'app/models/namespace/admin_note.rb' - 'app/models/namespace/aggregation_schedule.rb' + - 'app/models/namespace/detail.rb' - 'app/models/namespace/package_setting.rb' - 'app/models/namespace/root_storage_statistics.rb' - 'app/models/namespaces/sync_event.rb' diff --git a/app/models/namespace.rb b/app/models/namespace.rb index eb3108455840a2..06f49f16d66002 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -50,6 +50,7 @@ class Namespace < ApplicationRecord has_many :project_statistics has_one :namespace_settings, inverse_of: :namespace, class_name: 'NamespaceSetting', autosave: true has_one :ci_cd_settings, inverse_of: :namespace, class_name: 'NamespaceCiCdSetting', autosave: true + has_one :namespace_details, inverse_of: :namespace, class_name: 'Namespace::Detail', autosave: true has_one :namespace_statistics has_one :namespace_route, foreign_key: :namespace_id, autosave: false, inverse_of: :namespace, class_name: 'Route' has_many :namespace_members, foreign_key: :member_namespace_id, inverse_of: :member_namespace, class_name: 'Member' @@ -127,6 +128,7 @@ class Namespace < ApplicationRecord to: :namespace_settings, allow_nil: true after_save :schedule_sync_event_worker, if: -> { saved_change_to_id? || saved_change_to_parent_id? } + after_save :reload_namespace_details after_commit :refresh_access_of_projects_invited_groups, on: :update, if: -> { previous_changes.key?('share_with_group_lock') } @@ -676,6 +678,12 @@ def validate_parent_type end end + def reload_namespace_details + return unless !project_namespace? && (previous_changes.keys & %w(description description_html cached_markdown_version)).any? && namespace_details.present? + + namespace_details.reset + end + def sync_share_with_group_lock_with_parent if parent&.share_with_group_lock? self.share_with_group_lock = true diff --git a/app/models/namespace/detail.rb b/app/models/namespace/detail.rb new file mode 100644 index 00000000000000..dbbf9f4944ae84 --- /dev/null +++ b/app/models/namespace/detail.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +class Namespace::Detail < ApplicationRecord + belongs_to :namespace, inverse_of: :namespace_details + validates :namespace, presence: true + validates :description, length: { maximum: 255 } + + self.primary_key = :namespace_id +end diff --git a/app/models/project.rb b/app/models/project.rb index 1a30a2fe4ecbce..539e4bfabecd22 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -131,6 +131,8 @@ class Project < ApplicationRecord after_save :save_topics + after_save :reload_project_namespace_details + after_create -> { create_or_load_association(:project_feature) } after_create -> { create_or_load_association(:ci_cd_settings) } @@ -3257,6 +3259,12 @@ def sync_attributes(project_namespace) project_namespace.assign_attributes(attributes_to_sync) end + def reload_project_namespace_details + return unless (previous_changes.keys & %w(description description_html cached_markdown_version)).any? && project_namespace.namespace_details.present? + + project_namespace.namespace_details.reset + end + # SyncEvents are created by PG triggers (with the function `insert_projects_sync_event`) def schedule_sync_event_worker run_after_commit do diff --git a/db/docs/namespace_details.yml b/db/docs/namespace_details.yml new file mode 100644 index 00000000000000..f35bf7e3d85733 --- /dev/null +++ b/db/docs/namespace_details.yml @@ -0,0 +1,9 @@ +--- +table_name: namespace_details +classes: +- NamespaceDetail +feature_categories: +- subgroups +description: Used to store details for namespaces +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82958 +milestone: '15.2' diff --git a/db/migrate/20220316022505_create_namespace_details.rb b/db/migrate/20220316022505_create_namespace_details.rb index 2f3c3361493d3f..6df8606c726447 100644 --- a/db/migrate/20220316022505_create_namespace_details.rb +++ b/db/migrate/20220316022505_create_namespace_details.rb @@ -1,30 +1,21 @@ # frozen_string_literal: true -# See https://docs.gitlab.com/ee/development/migration_style_guide.html -# for more information on how to write migrations for GitLab. +class CreateNamespaceDetails < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! -class CreateNamespaceDetails < Gitlab::Database::Migration[1.0] - # When using the methods "add_concurrent_index" or "remove_concurrent_index" - # you must disable the use of transactions - # as these methods can not run in an existing transaction. - # When using "add_concurrent_index" or "remove_concurrent_index" methods make sure - # that either of them is the _only_ method called in the migration, - # any other changes should go in a separate migration. - # This ensures that upon failure _only_ the index creation or removing fails - # and can be retried or reverted easily. - # - # To disable transactions uncomment the following line and remove these - # comments: - # disable_ddl_transaction! - - def change - create_table :namespace_details do |t| - t.integer :namespace_id - t.string :description - t.text :description_html - t.integer :cached_markdown_version - - t.timestamps null: false + def up + with_lock_retries do + create_table :namespace_details, id: false do |t| + t.references :namespace, primary_key: true, null: false, default: nil, type: :bigint, index: false, foreign_key: { on_delete: :cascade } # rubocop:disable Layout/LineLength + t.timestamps_with_timezone null: true + t.integer :cached_markdown_version + t.text :description, limit: 255 + t.text :description_html, limit: 255 + end end end + + def down + drop_table :namespace_details + end end diff --git a/db/migrate/20220506154054_create_sync_namespace_details_trigger.rb b/db/migrate/20220506154054_create_sync_namespace_details_trigger.rb new file mode 100644 index 00000000000000..37bf9f8d9874b8 --- /dev/null +++ b/db/migrate/20220506154054_create_sync_namespace_details_trigger.rb @@ -0,0 +1,68 @@ +# frozen_string_literal: true +class CreateSyncNamespaceDetailsTrigger < Gitlab::Database::Migration[2.0] + include Gitlab::Database::SchemaHelpers + + UPDATE_TRIGGER_NAME = 'trigger_update_details_on_namespace_update' + INSERT_TRIGGER_NAME = 'trigger_update_details_on_namespace_insert' + FUNCTION_NAME = 'update_namespace_details_from_namespaces' + + def up + create_trigger_function(FUNCTION_NAME, replace: true) do + <<~SQL + INSERT INTO + namespace_details ( + description, + description_html, + cached_markdown_version, + updated_at, + created_at, + namespace_id + ) + VALUES + ( + NEW.description, + NEW.description_html, + NEW.cached_markdown_version, + NEW.updated_at, + NEW.updated_at, + NEW.id + ) ON CONFLICT (namespace_id) DO + UPDATE + SET + description = NEW.description, + description_html = NEW.description_html, + cached_markdown_version = NEW.cached_markdown_version, + updated_at = NEW.updated_at + WHERE + namespace_details.namespace_id = NEW.id;RETURN NULL; + SQL + end + + execute(<<~SQL) + CREATE TRIGGER #{UPDATE_TRIGGER_NAME} + AFTER UPDATE ON namespaces + FOR EACH ROW + WHEN ( + NEW.type <> 'Project' AND ( + OLD.description IS DISTINCT FROM NEW.description OR + OLD.description_html IS DISTINCT FROM NEW.description_html OR + OLD.cached_markdown_version IS DISTINCT FROM NEW.cached_markdown_version) + ) + EXECUTE PROCEDURE #{FUNCTION_NAME}(); + SQL + + execute(<<~SQL) + CREATE TRIGGER #{INSERT_TRIGGER_NAME} + AFTER INSERT ON namespaces + FOR EACH ROW + WHEN (NEW.type <> 'Project') + EXECUTE PROCEDURE #{FUNCTION_NAME}(); + SQL + end + + def down + drop_trigger(:namespaces, UPDATE_TRIGGER_NAME) + drop_trigger(:namespaces, INSERT_TRIGGER_NAME) + drop_function(FUNCTION_NAME) + end +end diff --git a/db/migrate/20220524184149_create_sync_project_namespace_details_trigger.rb b/db/migrate/20220524184149_create_sync_project_namespace_details_trigger.rb new file mode 100644 index 00000000000000..1d6e565c9c93a2 --- /dev/null +++ b/db/migrate/20220524184149_create_sync_project_namespace_details_trigger.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true +class CreateSyncProjectNamespaceDetailsTrigger < Gitlab::Database::Migration[2.0] + include Gitlab::Database::SchemaHelpers + + UPDATE_TRIGGER_NAME = 'trigger_update_details_on_project_update' + INSERT_TRIGGER_NAME = 'trigger_update_details_on_project_insert' + FUNCTION_NAME = 'update_namespace_details_from_projects' + + def up + create_trigger_function(FUNCTION_NAME, replace: true) do + <<~SQL + INSERT INTO + namespace_details ( + description, + description_html, + cached_markdown_version, + updated_at, + created_at, + namespace_id + ) + VALUES + ( + NEW.description, + NEW.description_html, + NEW.cached_markdown_version, + NEW.updated_at, + NEW.updated_at, + NEW.project_namespace_id + ) ON CONFLICT (namespace_id) DO + UPDATE + SET + description = NEW.description, + description_html = NEW.description_html, + cached_markdown_version = NEW.cached_markdown_version, + updated_at = NEW.updated_at + WHERE + namespace_details.namespace_id = NEW.project_namespace_id;RETURN NULL; + SQL + end + + execute(<<~SQL) + CREATE TRIGGER #{UPDATE_TRIGGER_NAME} + AFTER UPDATE ON projects + FOR EACH ROW + WHEN ( + OLD.description IS DISTINCT FROM NEW.description OR + OLD.description_html IS DISTINCT FROM NEW.description_html OR + OLD.cached_markdown_version IS DISTINCT FROM NEW.cached_markdown_version + ) + EXECUTE PROCEDURE #{FUNCTION_NAME}(); + SQL + + execute(<<~SQL) + CREATE TRIGGER #{INSERT_TRIGGER_NAME} + AFTER INSERT ON projects + FOR EACH ROW + EXECUTE PROCEDURE #{FUNCTION_NAME}(); + SQL + end + + def down + drop_trigger(:projects, UPDATE_TRIGGER_NAME) + drop_trigger(:projects, INSERT_TRIGGER_NAME) + drop_function(FUNCTION_NAME) + end +end diff --git a/db/schema_migrations/20220506154054 b/db/schema_migrations/20220506154054 new file mode 100644 index 00000000000000..8240d040c2549f --- /dev/null +++ b/db/schema_migrations/20220506154054 @@ -0,0 +1 @@ +a931441890bd5d472f88dcef82bb42e3c8046a981788f2362a8deb89f4ac049a \ No newline at end of file diff --git a/db/schema_migrations/20220524184149 b/db/schema_migrations/20220524184149 new file mode 100644 index 00000000000000..b75a7640a7651d --- /dev/null +++ b/db/schema_migrations/20220524184149 @@ -0,0 +1 @@ +f28bf2a6fe412342eef053b57cce14c0681d04f9e978e37bbd505f1efa36e92e \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index a1785a551d482d..ddcba7870e176a 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -259,6 +259,74 @@ RETURN NULL; END $$; +CREATE FUNCTION update_namespace_details_from_namespaces() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +INSERT INTO + namespace_details ( + description, + description_html, + cached_markdown_version, + updated_at, + created_at, + namespace_id + ) +VALUES + ( + NEW.description, + NEW.description_html, + NEW.cached_markdown_version, + NEW.updated_at, + NEW.updated_at, + NEW.id + ) ON CONFLICT (namespace_id) DO +UPDATE +SET + description = NEW.description, + description_html = NEW.description_html, + cached_markdown_version = NEW.cached_markdown_version, + updated_at = NEW.updated_at +WHERE + namespace_details.namespace_id = NEW.id;RETURN NULL; + +END +$$; + +CREATE FUNCTION update_namespace_details_from_projects() RETURNS trigger + LANGUAGE plpgsql + AS $$ +BEGIN +INSERT INTO + namespace_details ( + description, + description_html, + cached_markdown_version, + updated_at, + created_at, + namespace_id + ) +VALUES + ( + NEW.description, + NEW.description_html, + NEW.cached_markdown_version, + NEW.updated_at, + NEW.updated_at, + NEW.project_namespace_id + ) ON CONFLICT (namespace_id) DO +UPDATE +SET + description = NEW.description, + description_html = NEW.description_html, + cached_markdown_version = NEW.cached_markdown_version, + updated_at = NEW.updated_at +WHERE + namespace_details.namespace_id = NEW.project_namespace_id;RETURN NULL; + +END +$$; + CREATE FUNCTION update_vulnerability_reads_from_vulnerability() RETURNS trigger LANGUAGE plpgsql AS $$ @@ -17556,24 +17624,16 @@ CREATE TABLE namespace_ci_cd_settings ( ); CREATE TABLE namespace_details ( - id bigint NOT NULL, - namespace_id integer, - description character varying, - description_html text, + namespace_id bigint NOT NULL, + created_at timestamp with time zone, + updated_at timestamp with time zone, cached_markdown_version integer, - created_at timestamp(6) without time zone NOT NULL, - updated_at timestamp(6) without time zone NOT NULL + description text, + description_html text, + CONSTRAINT check_2df620eaf6 CHECK ((char_length(description_html) <= 255)), + CONSTRAINT check_2f563eec0f CHECK ((char_length(description) <= 255)) ); -CREATE SEQUENCE namespace_details_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE namespace_details_id_seq OWNED BY namespace_details.id; - CREATE TABLE namespace_limits ( additional_purchased_storage_size bigint DEFAULT 0 NOT NULL, additional_purchased_storage_ends_on date, @@ -25391,8 +25451,9 @@ ALTER TABLE ONLY namespace_bans ALTER TABLE ONLY namespace_ci_cd_settings ADD CONSTRAINT namespace_ci_cd_settings_pkey PRIMARY KEY (namespace_id); + ALTER TABLE ONLY namespace_details - ADD CONSTRAINT namespace_details_pkey PRIMARY KEY (id); + ADD CONSTRAINT namespace_details_pkey PRIMARY KEY (namespace_id); ALTER TABLE ONLY namespace_limits ADD CONSTRAINT namespace_limits_pkey PRIMARY KEY (namespace_id); @@ -31766,6 +31827,14 @@ CREATE TRIGGER trigger_projects_parent_id_on_insert AFTER INSERT ON projects FOR CREATE TRIGGER trigger_projects_parent_id_on_update AFTER UPDATE ON projects FOR EACH ROW WHEN ((old.namespace_id IS DISTINCT FROM new.namespace_id)) EXECUTE FUNCTION insert_projects_sync_event(); +CREATE TRIGGER trigger_update_details_on_namespace_insert AFTER INSERT ON namespaces FOR EACH ROW WHEN (((new.type)::text <> 'Project'::text)) EXECUTE FUNCTION update_namespace_details_from_namespaces(); + +CREATE TRIGGER trigger_update_details_on_namespace_update AFTER UPDATE ON namespaces FOR EACH ROW WHEN ((((new.type)::text <> 'Project'::text) AND (((old.description)::text IS DISTINCT FROM (new.description)::text) OR (old.description_html IS DISTINCT FROM new.description_html) OR (old.cached_markdown_version IS DISTINCT FROM new.cached_markdown_version)))) EXECUTE FUNCTION update_namespace_details_from_namespaces(); + +CREATE TRIGGER trigger_update_details_on_project_insert AFTER INSERT ON projects FOR EACH ROW EXECUTE FUNCTION update_namespace_details_from_projects(); + +CREATE TRIGGER trigger_update_details_on_project_update AFTER UPDATE ON projects FOR EACH ROW WHEN (((old.description IS DISTINCT FROM new.description) OR (old.description_html IS DISTINCT FROM new.description_html) OR (old.cached_markdown_version IS DISTINCT FROM new.cached_markdown_version))) EXECUTE FUNCTION update_namespace_details_from_projects(); + CREATE TRIGGER trigger_update_has_issues_on_vulnerability_issue_links_delete AFTER DELETE ON vulnerability_issue_links FOR EACH ROW EXECUTE FUNCTION unset_has_issues_on_vulnerability_reads(); CREATE TRIGGER trigger_update_has_issues_on_vulnerability_issue_links_update AFTER INSERT ON vulnerability_issue_links FOR EACH ROW EXECUTE FUNCTION set_has_issues_on_vulnerability_reads(); diff --git a/lib/gitlab/database/gitlab_schemas.yml b/lib/gitlab/database/gitlab_schemas.yml index 26bdcdc8da3695..a99f6585d9c759 100644 --- a/lib/gitlab/database/gitlab_schemas.yml +++ b/lib/gitlab/database/gitlab_schemas.yml @@ -332,6 +332,7 @@ namespace_package_settings: :gitlab_main namespace_root_storage_statistics: :gitlab_main namespace_ci_cd_settings: :gitlab_main namespace_settings: :gitlab_main +namespace_details: :gitlab_main namespaces: :gitlab_main namespaces_sync_events: :gitlab_main namespace_statistics: :gitlab_main diff --git a/spec/migrations/20220506154054_create_sync_namespace_details_trigger_spec.rb b/spec/migrations/20220506154054_create_sync_namespace_details_trigger_spec.rb new file mode 100644 index 00000000000000..aedea6811978c6 --- /dev/null +++ b/spec/migrations/20220506154054_create_sync_namespace_details_trigger_spec.rb @@ -0,0 +1,76 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_migration! + +RSpec.describe CreateSyncNamespaceDetailsTrigger do + let(:migration) { described_class.new } + let(:namespaces) { table(:namespaces) } + let(:namespace_details) { table(:namespace_details) } + let!(:timestamp) { Time.new(2020, 01, 01).utc } + + let(:synced_attributes) do + { + description: 'description', + description_html: '

description

', + cached_markdown_version: 1966080, + created_at: timestamp, + updated_at: timestamp + } + end + + let(:other_attributes) do + { + name: 'name', + path: 'path' + } + end + + let(:attributes) { other_attributes.merge(synced_attributes) } + + describe '#up' do + before do + migrate! + end + + describe 'INSERT trigger' do + it 'creates a namespace_detail record' do + expect do + namespaces.create!(attributes) + end.to change { namespace_details.count }.by(1) + end + + it 'the created namespace_details record has matching attributes' do + namespaces.create!(attributes) + synced_namespace_details = namespace_details.last + + expect(synced_namespace_details).to have_attributes(synced_attributes) + end + end + + describe 'UPDATE trigger' do + let!(:namespace) { namespaces.create!(attributes) } + + it 'updates the attribute in the synced namespace_details record' do + namespace.update!(description: 'new_description') + + synced_namespace_details = namespace_details.last + expect(synced_namespace_details.description).to eq('new_description') + end + end + end + + describe '#down' do + before do + migration.up + migration.down + end + + it 'drops the trigger' do + expect do + namespaces.create!(attributes) + end.not_to change { namespace_details.count } + end + end +end diff --git a/spec/migrations/20220524184149_create_sync_project_namespace_details_trigger_spec.rb b/spec/migrations/20220524184149_create_sync_project_namespace_details_trigger_spec.rb new file mode 100644 index 00000000000000..aa0e351103c898 --- /dev/null +++ b/spec/migrations/20220524184149_create_sync_project_namespace_details_trigger_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'spec_helper' + +require_migration! + +RSpec.describe CreateSyncProjectNamespaceDetailsTrigger do + let(:migration) { described_class.new } + let(:projects) { table(:projects) } + let(:namespaces) { table(:namespaces) } + let(:namespace_details) { table(:namespace_details) } + let!(:timestamp) { Time.new(2020, 01, 01).utc } + let!(:project_namespace) { namespaces.create!(name: 'name', path: 'path') } + let!(:namespace) { namespaces.create!(name: 'group', path: 'group_path') } + + let(:synced_attributes) do + { + description: 'description', + description_html: '

description

', + cached_markdown_version: 1966080, + updated_at: timestamp + } + end + + let(:other_attributes) do + { + name: 'project_name', + project_namespace_id: project_namespace.id, + namespace_id: namespace.id + } + end + + let(:attributes) { other_attributes.merge(synced_attributes) } + + describe '#up' do + before do + migrate! + end + + describe 'INSERT trigger' do + it 'the created namespace_details record has matching attributes' do + project = projects.create!(attributes) + synced_namespace_details = namespace_details.find_by(namespace_id: project.project_namespace_id) + + expect(synced_namespace_details).to have_attributes(synced_attributes) + end + end + + describe 'UPDATE trigger' do + let!(:project) { projects.create!(attributes) } + + it 'updates the attribute in the synced namespace_details record' do + project.update!(description: 'new_description') + + synced_namespace_details = namespace_details.find_by(namespace_id: project.project_namespace_id) + expect(synced_namespace_details.description).to eq('new_description') + end + end + end + + describe '#down' do + before do + migration.up + migration.down + end + + it 'drops the trigger' do + expect do + projects.create!(attributes) + end.not_to change { namespace_details.count } + end + end +end diff --git a/spec/models/namespace/detail_spec.rb b/spec/models/namespace/detail_spec.rb new file mode 100644 index 00000000000000..1bb756c441ba2a --- /dev/null +++ b/spec/models/namespace/detail_spec.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Namespace::Detail, type: :model do + describe 'associations' do + it { is_expected.to belong_to :namespace } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:namespace) } + end + + context 'when namespace description changes' do + let(:namespace) { create(:namespace, description: "old") } + + it 'changes namespace details description' do + expect { namespace.update!(description: "new") } + .to change { namespace.namespace_details.description }.from("old").to("new") + end + end + + context 'when project description changes' do + let(:project) { create(:project, description: "old") } + + it 'changes project namespace details description' do + expect { project.update!(description: "new") } + .to change { project.project_namespace.namespace_details.description }.from("old").to("new") + end + end + + context 'when group description changes' do + let(:group) { create(:group, description: "old") } + + it 'changes group namespace details description' do + expect { group.update!(description: "new") } + .to change { group.namespace_details.description }.from("old").to("new") + end + end +end diff --git a/spec/models/namespace_spec.rb b/spec/models/namespace_spec.rb index 2a5794b31d2336..f9441a7a5e12af 100644 --- a/spec/models/namespace_spec.rb +++ b/spec/models/namespace_spec.rb @@ -22,6 +22,7 @@ it { is_expected.to have_one :root_storage_statistics } it { is_expected.to have_one :aggregation_schedule } it { is_expected.to have_one :namespace_settings } + it { is_expected.to have_one :namespace_details } it { is_expected.to have_one(:namespace_statistics) } it { is_expected.to have_many :custom_emoji } it { is_expected.to have_one :package_setting_relation } diff --git a/spec/requests/api/project_import_spec.rb b/spec/requests/api/project_import_spec.rb index 8655e5b02387b5..afe5a7d4a21ac1 100644 --- a/spec/requests/api/project_import_spec.rb +++ b/spec/requests/api/project_import_spec.rb @@ -47,7 +47,7 @@ it 'executes a limited number of queries' do control_count = ActiveRecord::QueryRecorder.new { subject }.count - expect(control_count).to be <= 108 + expect(control_count).to be <= 109 end it 'schedules an import using a namespace' do diff --git a/spec/services/groups/create_service_spec.rb b/spec/services/groups/create_service_spec.rb index 6e074f451c4f0c..0cfde9ef434c7f 100644 --- a/spec/services/groups/create_service_spec.rb +++ b/spec/services/groups/create_service_spec.rb @@ -176,6 +176,15 @@ end end + describe 'creating a details record' do + let(:service) { described_class.new(user, group_params) } + + it 'create the details record connected to the group' do + group = subject + expect(group.namespace_details).to be_persisted + end + end + describe 'create service for the group' do let(:service) { described_class.new(user, group_params) } let(:created_group) { service.execute } -- GitLab From a0a7beb0e7e0e3f4b9fa907516ca32a3380667b8 Mon Sep 17 00:00:00 2001 From: Serena Fang Date: Wed, 20 Jul 2022 14:00:25 -0500 Subject: [PATCH 3/8] Update mile stone --- db/docs/namespace_details.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/namespace_details.yml b/db/docs/namespace_details.yml index f35bf7e3d85733..00053d3939649d 100644 --- a/db/docs/namespace_details.yml +++ b/db/docs/namespace_details.yml @@ -6,4 +6,4 @@ feature_categories: - subgroups description: Used to store details for namespaces introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/82958 -milestone: '15.2' +milestone: '15.3' -- GitLab From e181bc8e918527821d32c904a25eb7614fdaa8a4 Mon Sep 17 00:00:00 2001 From: Serena Fang Date: Mon, 1 Aug 2022 17:20:28 -0500 Subject: [PATCH 4/8] Enable lock retries --- .../20220506154054_create_sync_namespace_details_trigger.rb | 2 ++ ...20524184149_create_sync_project_namespace_details_trigger.rb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/db/migrate/20220506154054_create_sync_namespace_details_trigger.rb b/db/migrate/20220506154054_create_sync_namespace_details_trigger.rb index 37bf9f8d9874b8..1351fe91318de0 100644 --- a/db/migrate/20220506154054_create_sync_namespace_details_trigger.rb +++ b/db/migrate/20220506154054_create_sync_namespace_details_trigger.rb @@ -6,6 +6,8 @@ class CreateSyncNamespaceDetailsTrigger < Gitlab::Database::Migration[2.0] INSERT_TRIGGER_NAME = 'trigger_update_details_on_namespace_insert' FUNCTION_NAME = 'update_namespace_details_from_namespaces' + enable_lock_retries! + def up create_trigger_function(FUNCTION_NAME, replace: true) do <<~SQL diff --git a/db/migrate/20220524184149_create_sync_project_namespace_details_trigger.rb b/db/migrate/20220524184149_create_sync_project_namespace_details_trigger.rb index 1d6e565c9c93a2..efce35b443ac49 100644 --- a/db/migrate/20220524184149_create_sync_project_namespace_details_trigger.rb +++ b/db/migrate/20220524184149_create_sync_project_namespace_details_trigger.rb @@ -6,6 +6,8 @@ class CreateSyncProjectNamespaceDetailsTrigger < Gitlab::Database::Migration[2.0 INSERT_TRIGGER_NAME = 'trigger_update_details_on_project_insert' FUNCTION_NAME = 'update_namespace_details_from_projects' + enable_lock_retries! + def up create_trigger_function(FUNCTION_NAME, replace: true) do <<~SQL -- GitLab From 42101996f96961238f1801c2a3f90318acb80c58 Mon Sep 17 00:00:00 2001 From: Serena Fang Date: Tue, 15 Mar 2022 21:27:44 -0500 Subject: [PATCH 5/8] Create NamespaceDetails table Changelog: changed Add namespace details feature flag Prepare migrations for NamespaceDetail Todo: figure out backfilling Update create namespace detail table Add post migration file Fix rubocop errors Fix rubocop error Add backfill namespace details spec Run rails migrate Add namespace details to gitlab schemas Select other columns Remove schema version number Fix rubocop errors Rename files Add namespace scope Fix table name Fix rspec errors Remove unnecessary change Move spec file Remove extra file Run rails migrate Run rails migrate Fix db structure Rename NamespaceDetail to be in module NamespaceDetail -> Namespace::Detail Sync namespace details Remove extra file Update project namespace details when project description changes Move namespace details to new method Remove extra attributes Remove extra line Check if project namespace present Create backfill project namespace details Need migration to backfill project namespace details Backfill project namespace details spec Fix specs Fix specs Add db docs Ignore project namespaces Apply reviewer suggestions Add feature flag Remove trailing whitespace Rebase with master Loose foreign keys Update ff milestone Move backfill files Remove schema migrations Add description validation Add namespace details trigger Add database trigger testing my network Revert "testing my network" This reverts commit 81e5160cc0b2293c220bb36e381259e523d95917. Run rubocop linter Remove create trigger Add new line Apply reviewer suggestions Add insert trigger Fix structure sql Add project namespace trigger Update db structure Fix some of the failures Fix some of the failures Add cr remarks Check for project type Check namespace type Change trigger sql Update db structure Update trigger code Update db schema Do not reload project namespace Change db structure Change db structure Fix db structure Fix db structure Fix db structure Remove extra spaces Update mile stone Enable lock retries Fix structure sql Fix structure sql Remove extra line --- db/structure.sql | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/db/structure.sql b/db/structure.sql index ddcba7870e176a..18408dcf3ee7c8 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -23433,8 +23433,6 @@ ALTER TABLE ONLY namespace_admin_notes ALTER COLUMN id SET DEFAULT nextval('name ALTER TABLE ONLY namespace_bans ALTER COLUMN id SET DEFAULT nextval('namespace_bans_id_seq'::regclass); -ALTER TABLE ONLY namespace_details ALTER COLUMN id SET DEFAULT nextval('namespace_details_id_seq'::regclass); - ALTER TABLE ONLY namespace_statistics ALTER COLUMN id SET DEFAULT nextval('namespace_statistics_id_seq'::regclass); ALTER TABLE ONLY namespaces ALTER COLUMN id SET DEFAULT nextval('namespaces_id_seq'::regclass); @@ -25455,6 +25453,9 @@ ALTER TABLE ONLY namespace_ci_cd_settings ALTER TABLE ONLY namespace_details ADD CONSTRAINT namespace_details_pkey PRIMARY KEY (namespace_id); +ALTER TABLE ONLY namespace_details + ADD CONSTRAINT namespace_details_pkey PRIMARY KEY (namespace_id); + ALTER TABLE ONLY namespace_limits ADD CONSTRAINT namespace_limits_pkey PRIMARY KEY (namespace_id); @@ -34029,6 +34030,9 @@ ALTER TABLE ONLY boards_epic_board_positions ALTER TABLE ONLY vulnerability_finding_links ADD CONSTRAINT fk_rails_cbdfde27ce FOREIGN KEY (vulnerability_occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE; +ALTER TABLE ONLY namespace_details + ADD CONSTRAINT fk_rails_cc11a451f8 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; + ALTER TABLE ONLY issues_self_managed_prometheus_alert_events ADD CONSTRAINT fk_rails_cc5d88bbb0 FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE; -- GitLab From 532a367e666db5afe8f4c44e2668082c315d67eb Mon Sep 17 00:00:00 2001 From: Serena Fang Date: Tue, 2 Aug 2022 19:19:55 -0500 Subject: [PATCH 6/8] Remove extra line --- db/structure.sql | 3 --- 1 file changed, 3 deletions(-) diff --git a/db/structure.sql b/db/structure.sql index 18408dcf3ee7c8..5fe86e1d114658 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -25453,9 +25453,6 @@ ALTER TABLE ONLY namespace_ci_cd_settings ALTER TABLE ONLY namespace_details ADD CONSTRAINT namespace_details_pkey PRIMARY KEY (namespace_id); -ALTER TABLE ONLY namespace_details - ADD CONSTRAINT namespace_details_pkey PRIMARY KEY (namespace_id); - ALTER TABLE ONLY namespace_limits ADD CONSTRAINT namespace_limits_pkey PRIMARY KEY (namespace_id); -- GitLab From 4040c2908629b8b6e9ca5e89a8a95e1833db683a Mon Sep 17 00:00:00 2001 From: Serena Fang Date: Tue, 2 Aug 2022 23:19:14 -0500 Subject: [PATCH 7/8] Fix rubocop error --- ...220506154054_create_sync_namespace_details_trigger_spec.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/spec/migrations/20220506154054_create_sync_namespace_details_trigger_spec.rb b/spec/migrations/20220506154054_create_sync_namespace_details_trigger_spec.rb index aedea6811978c6..411b1eacb8634d 100644 --- a/spec/migrations/20220506154054_create_sync_namespace_details_trigger_spec.rb +++ b/spec/migrations/20220506154054_create_sync_namespace_details_trigger_spec.rb @@ -38,7 +38,7 @@ it 'creates a namespace_detail record' do expect do namespaces.create!(attributes) - end.to change { namespace_details.count }.by(1) + end.to change(namespace_details, :count).by(1) end it 'the created namespace_details record has matching attributes' do @@ -70,7 +70,7 @@ it 'drops the trigger' do expect do namespaces.create!(attributes) - end.not_to change { namespace_details.count } + end.not_to change(namespace_details, :count) end end end -- GitLab From d1b33c9f7b979ca93b4a0d5636962b6cc63422a9 Mon Sep 17 00:00:00 2001 From: Serena Fang Date: Thu, 4 Aug 2022 14:46:22 -0500 Subject: [PATCH 8/8] Fix rubocop failure --- ...184149_create_sync_project_namespace_details_trigger_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spec/migrations/20220524184149_create_sync_project_namespace_details_trigger_spec.rb b/spec/migrations/20220524184149_create_sync_project_namespace_details_trigger_spec.rb index aa0e351103c898..f85a59357e1903 100644 --- a/spec/migrations/20220524184149_create_sync_project_namespace_details_trigger_spec.rb +++ b/spec/migrations/20220524184149_create_sync_project_namespace_details_trigger_spec.rb @@ -67,7 +67,7 @@ it 'drops the trigger' do expect do projects.create!(attributes) - end.not_to change { namespace_details.count } + end.not_to change(namespace_details, :count) end end end -- GitLab