From 0ddb2670a4b253d0b142a86ffeacd83def1e4488 Mon Sep 17 00:00:00 2001 From: lma-git Date: Tue, 27 Feb 2024 10:54:49 -0800 Subject: [PATCH 1/3] Add table to track CI component usage Adds a new partitioned table p_catalog_resource_component_usages. This table will track when a CI component is included in a project's pipeline. Only unique records of component-used_by_project-used_date are inserted. This data will be used to display component usage popularity aggregated by component or catalog_resource per 30 day rolling window. This table is partitioned by month. For now, the table will drop partitions with data older than 12 months. This duration will likely be reduced in a follow up issue. Changelog: added --- app/models/ci/catalog/resource.rb | 2 + app/models/ci/catalog/resources/component.rb | 7 +- .../ci/catalog/resources/components/usage.rb | 37 ++++++++++ app/models/project.rb | 1 + config/initializers/postgres_partitioning.rb | 3 +- .../p_catalog_resource_component_usages.yml | 12 +++ ...catalog_resource_component_usages_table.rb | 34 +++++++++ ...fk_to_catalog_resource_component_usages.rb | 20 +++++ ...fk_to_catalog_resource_component_usages.rb | 20 +++++ ...fk_to_catalog_resource_component_usages.rb | 20 +++++ db/schema_migrations/20240301210341 | 1 + db/schema_migrations/20240301210400 | 1 + db/schema_migrations/20240301210420 | 1 + db/schema_migrations/20240301210440 | 1 + db/structure.sql | 39 ++++++++++ ee/spec/models/factories_spec.rb | 1 + spec/db/schema_spec.rb | 1 + .../ci/catalog/resources/components/usages.rb | 10 +++ spec/lib/gitlab/import_export/all_models.yml | 1 + spec/models/ci/catalog/resource_spec.rb | 6 ++ .../ci/catalog/resources/component_spec.rb | 1 + .../resources/components/usage_spec.rb | 74 +++++++++++++++++++ spec/models/project_spec.rb | 1 + 23 files changed, 291 insertions(+), 3 deletions(-) create mode 100644 app/models/ci/catalog/resources/components/usage.rb create mode 100644 db/docs/p_catalog_resource_component_usages.yml create mode 100644 db/migrate/20240301210341_create_catalog_resource_component_usages_table.rb create mode 100644 db/migrate/20240301210400_add_component_fk_to_catalog_resource_component_usages.rb create mode 100644 db/migrate/20240301210420_add_catalog_resource_fk_to_catalog_resource_component_usages.rb create mode 100644 db/migrate/20240301210440_add_project_fk_to_catalog_resource_component_usages.rb create mode 100644 db/schema_migrations/20240301210341 create mode 100644 db/schema_migrations/20240301210400 create mode 100644 db/schema_migrations/20240301210420 create mode 100644 db/schema_migrations/20240301210440 create mode 100644 spec/factories/ci/catalog/resources/components/usages.rb create mode 100644 spec/models/ci/catalog/resources/components/usage_spec.rb diff --git a/app/models/ci/catalog/resource.rb b/app/models/ci/catalog/resource.rb index a4535a2a17ddfe..32f469ac265613 100644 --- a/app/models/ci/catalog/resource.rb +++ b/app/models/ci/catalog/resource.rb @@ -17,6 +17,8 @@ class Resource < ::ApplicationRecord belongs_to :project has_many :components, class_name: 'Ci::Catalog::Resources::Component', foreign_key: :catalog_resource_id, inverse_of: :catalog_resource + has_many :component_usages, class_name: 'Ci::Catalog::Resources::Components::Usage', + foreign_key: :catalog_resource_id, inverse_of: :catalog_resource has_many :versions, class_name: 'Ci::Catalog::Resources::Version', foreign_key: :catalog_resource_id, inverse_of: :catalog_resource has_many :sync_events, class_name: 'Ci::Catalog::Resources::SyncEvent', foreign_key: :catalog_resource_id, diff --git a/app/models/ci/catalog/resources/component.rb b/app/models/ci/catalog/resources/component.rb index 07d5404981bacb..88e6f301c4853c 100644 --- a/app/models/ci/catalog/resources/component.rb +++ b/app/models/ci/catalog/resources/component.rb @@ -6,13 +6,16 @@ module Resources # This class represents a CI/CD Catalog resource component. # The data will be used as metadata of a component. class Component < ::ApplicationRecord - include BulkInsertSafe - self.table_name = 'catalog_resource_components' belongs_to :project, inverse_of: :ci_components belongs_to :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :components belongs_to :version, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :components + has_many :usages, class_name: 'Ci::Catalog::Resources::Components::Usage', inverse_of: :component + + # BulkInsertSafe must be included after the `has_many` declaration, otherwise it raises + # an error about the save callback that is auto generated for this association. + include BulkInsertSafe enum resource_type: { template: 1 } diff --git a/app/models/ci/catalog/resources/components/usage.rb b/app/models/ci/catalog/resources/components/usage.rb new file mode 100644 index 00000000000000..119e61ab11bbfa --- /dev/null +++ b/app/models/ci/catalog/resources/components/usage.rb @@ -0,0 +1,37 @@ +# frozen_string_literal: true + +module Ci + module Catalog + module Resources + module Components + # This model is used to track when a project includes a component in a pipeline. + # The column `used_by_project_id` does not have an FK constraint because we want + # to preserve historical usage data. + class Usage < ::ApplicationRecord + include PartitionedTable + + self.table_name = 'p_catalog_resource_component_usages' + self.primary_key = :id + + # TO DO: Retention period to be shortened in https://gitlab.com/gitlab-org/gitlab/-/issues/443681 + partitioned_by :used_date, strategy: :monthly, retain_for: 12.months + + belongs_to :component, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :usages + belongs_to :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :component_usages + belongs_to :project, inverse_of: :ci_component_usages + + validates :component, :catalog_resource, :project, :used_by_project_id, presence: true + validates :used_date, uniqueness: { scope: [:component_id, :used_by_project_id] } + + before_validation :set_used_date, unless: :used_date? + + private + + def set_used_date + self.used_date = Date.today + end + end + end + end + end +end diff --git a/app/models/project.rb b/app/models/project.rb index 70e9c82014a389..978c2520df4191 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -189,6 +189,7 @@ class Project < ApplicationRecord has_one :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :project has_many :ci_components, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :project + has_many :ci_component_usages, class_name: 'Ci::Catalog::Resources::Components::Usage', inverse_of: :project has_many :catalog_resource_versions, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :project has_many :catalog_resource_sync_events, class_name: 'Ci::Catalog::Resources::SyncEvent', inverse_of: :project diff --git a/config/initializers/postgres_partitioning.rb b/config/initializers/postgres_partitioning.rb index 56dd1cb43f1810..2b56e0eea191a6 100644 --- a/config/initializers/postgres_partitioning.rb +++ b/config/initializers/postgres_partitioning.rb @@ -14,7 +14,8 @@ BatchedGitRefUpdates::Deletion, Users::ProjectVisit, Users::GroupVisit, - Ci::Catalog::Resources::SyncEvent + Ci::Catalog::Resources::SyncEvent, + Ci::Catalog::Resources::Components::Usage ]) if Gitlab.ee? diff --git a/db/docs/p_catalog_resource_component_usages.yml b/db/docs/p_catalog_resource_component_usages.yml new file mode 100644 index 00000000000000..1d176b501f74c2 --- /dev/null +++ b/db/docs/p_catalog_resource_component_usages.yml @@ -0,0 +1,12 @@ +--- +table_name: p_catalog_resource_component_usages +classes: +- Ci::Catalog::Resources::Components::Usage +feature_categories: +- pipeline_composition +description: Tracks when a project includes a CI component in a pipeline. +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/145881 +milestone: '16.10' +gitlab_schema: gitlab_main_cell +sharding_key: + project_id: projects diff --git a/db/migrate/20240301210341_create_catalog_resource_component_usages_table.rb b/db/migrate/20240301210341_create_catalog_resource_component_usages_table.rb new file mode 100644 index 00000000000000..307068fc096917 --- /dev/null +++ b/db/migrate/20240301210341_create_catalog_resource_component_usages_table.rb @@ -0,0 +1,34 @@ +# frozen_string_literal: true + +class CreateCatalogResourceComponentUsagesTable < Gitlab::Database::Migration[2.2] + milestone '16.10' + + enable_lock_retries! + + CATALOG_RESOURCE_INDEX_NAME = 'idx_p_catalog_resource_component_usages_on_catalog_resource_id' + UNIQUE_INDEX_NAME = 'idx_component_usages_on_component_used_by_project_and_used_date' + + def up + options = { + primary_key: [:id, :used_date], + options: 'PARTITION BY RANGE (used_date)', + if_not_exists: true + } + + create_table(:p_catalog_resource_component_usages, **options) do |t| + t.bigserial :id, null: false + t.bigint :component_id, null: false + t.bigint :catalog_resource_id, null: false + t.bigint :project_id, null: false, index: true + t.bigint :used_by_project_id, null: false + t.date :used_date, null: false + + t.index :catalog_resource_id, name: CATALOG_RESOURCE_INDEX_NAME + t.index [:component_id, :used_by_project_id, :used_date], unique: true, name: UNIQUE_INDEX_NAME + end + end + + def down + drop_table :p_catalog_resource_component_usages + end +end diff --git a/db/migrate/20240301210400_add_component_fk_to_catalog_resource_component_usages.rb b/db/migrate/20240301210400_add_component_fk_to_catalog_resource_component_usages.rb new file mode 100644 index 00000000000000..5d524373596e0c --- /dev/null +++ b/db/migrate/20240301210400_add_component_fk_to_catalog_resource_component_usages.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddComponentFkToCatalogResourceComponentUsages < Gitlab::Database::Migration[2.2] + include Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers + + milestone '16.10' + + disable_ddl_transaction! + + def up + add_concurrent_partitioned_foreign_key :p_catalog_resource_component_usages, :catalog_resource_components, + column: :component_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :p_catalog_resource_component_usages, column: :component_id + end + end +end diff --git a/db/migrate/20240301210420_add_catalog_resource_fk_to_catalog_resource_component_usages.rb b/db/migrate/20240301210420_add_catalog_resource_fk_to_catalog_resource_component_usages.rb new file mode 100644 index 00000000000000..2ea97f7d74cc37 --- /dev/null +++ b/db/migrate/20240301210420_add_catalog_resource_fk_to_catalog_resource_component_usages.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddCatalogResourceFkToCatalogResourceComponentUsages < Gitlab::Database::Migration[2.2] + include Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers + + milestone '16.10' + + disable_ddl_transaction! + + def up + add_concurrent_partitioned_foreign_key :p_catalog_resource_component_usages, :catalog_resources, + column: :catalog_resource_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :p_catalog_resource_component_usages, column: :catalog_resource_id + end + end +end diff --git a/db/migrate/20240301210440_add_project_fk_to_catalog_resource_component_usages.rb b/db/migrate/20240301210440_add_project_fk_to_catalog_resource_component_usages.rb new file mode 100644 index 00000000000000..5056c748d1505d --- /dev/null +++ b/db/migrate/20240301210440_add_project_fk_to_catalog_resource_component_usages.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +class AddProjectFkToCatalogResourceComponentUsages < Gitlab::Database::Migration[2.2] + include Gitlab::Database::PartitioningMigrationHelpers::ForeignKeyHelpers + + milestone '16.10' + + disable_ddl_transaction! + + def up + add_concurrent_partitioned_foreign_key :p_catalog_resource_component_usages, :projects, + column: :project_id, on_delete: :cascade + end + + def down + with_lock_retries do + remove_foreign_key :p_catalog_resource_component_usages, column: :project_id + end + end +end diff --git a/db/schema_migrations/20240301210341 b/db/schema_migrations/20240301210341 new file mode 100644 index 00000000000000..af61e0549cd2d6 --- /dev/null +++ b/db/schema_migrations/20240301210341 @@ -0,0 +1 @@ +e7f5750d0c275f509cffa49a1365c9fff53362bf5b9f7239a1be5c67cb62273c \ No newline at end of file diff --git a/db/schema_migrations/20240301210400 b/db/schema_migrations/20240301210400 new file mode 100644 index 00000000000000..356410a04eb356 --- /dev/null +++ b/db/schema_migrations/20240301210400 @@ -0,0 +1 @@ +1b8a8f741ac2fcf7088e5f4ab53d725cbdda3b69665b8bd128d7292ec4f396fd \ No newline at end of file diff --git a/db/schema_migrations/20240301210420 b/db/schema_migrations/20240301210420 new file mode 100644 index 00000000000000..964f42380ffee6 --- /dev/null +++ b/db/schema_migrations/20240301210420 @@ -0,0 +1 @@ +f7e58e44184c7591938373d2024d5d1a1192695f32272e09e24da12944f6c108 \ No newline at end of file diff --git a/db/schema_migrations/20240301210440 b/db/schema_migrations/20240301210440 new file mode 100644 index 00000000000000..0b22d0dba87c45 --- /dev/null +++ b/db/schema_migrations/20240301210440 @@ -0,0 +1 @@ +748b812798302fd55a6696ad37d79ae42634f499a3c0733b7b7ec6089424182f \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 4ad48e8acdefb2..9f0fbaa2ebd2b2 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12174,6 +12174,25 @@ CREATE SEQUENCE p_batched_git_ref_updates_deletions_id_seq ALTER SEQUENCE p_batched_git_ref_updates_deletions_id_seq OWNED BY p_batched_git_ref_updates_deletions.id; +CREATE TABLE p_catalog_resource_component_usages ( + id bigint NOT NULL, + component_id bigint NOT NULL, + catalog_resource_id bigint NOT NULL, + project_id bigint NOT NULL, + used_by_project_id bigint NOT NULL, + used_date date NOT NULL +) +PARTITION BY RANGE (used_date); + +CREATE SEQUENCE p_catalog_resource_component_usages_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE p_catalog_resource_component_usages_id_seq OWNED BY p_catalog_resource_component_usages.id; + CREATE SEQUENCE p_catalog_resource_sync_events_id_seq START WITH 1 INCREMENT BY 1 @@ -19140,6 +19159,8 @@ ALTER TABLE ONLY organizations ALTER COLUMN id SET DEFAULT nextval('organization ALTER TABLE ONLY p_batched_git_ref_updates_deletions ALTER COLUMN id SET DEFAULT nextval('p_batched_git_ref_updates_deletions_id_seq'::regclass); +ALTER TABLE ONLY p_catalog_resource_component_usages ALTER COLUMN id SET DEFAULT nextval('p_catalog_resource_component_usages_id_seq'::regclass); + ALTER TABLE ONLY p_catalog_resource_sync_events ALTER COLUMN id SET DEFAULT nextval('p_catalog_resource_sync_events_id_seq'::regclass); ALTER TABLE ONLY p_ci_builds_metadata ALTER COLUMN id SET DEFAULT nextval('ci_builds_metadata_id_seq'::regclass); @@ -21447,6 +21468,9 @@ ALTER TABLE ONLY organizations ALTER TABLE ONLY p_batched_git_ref_updates_deletions ADD CONSTRAINT p_batched_git_ref_updates_deletions_pkey PRIMARY KEY (id, partition_id); +ALTER TABLE ONLY p_catalog_resource_component_usages + ADD CONSTRAINT p_catalog_resource_component_usages_pkey PRIMARY KEY (id, used_date); + ALTER TABLE ONLY p_catalog_resource_sync_events ADD CONSTRAINT p_catalog_resource_sync_events_pkey PRIMARY KEY (id, partition_id); @@ -23554,6 +23578,8 @@ CREATE INDEX idx_ci_pipelines_artifacts_locked ON ci_pipelines USING btree (ci_r CREATE INDEX idx_compliance_security_policies_on_policy_configuration_id ON compliance_framework_security_policies USING btree (policy_configuration_id); +CREATE UNIQUE INDEX idx_component_usages_on_component_used_by_project_and_used_date ON ONLY p_catalog_resource_component_usages USING btree (component_id, used_by_project_id, used_date); + CREATE INDEX idx_container_exp_policies_on_project_id_next_run_at ON container_expiration_policies USING btree (project_id, next_run_at) WHERE (enabled = true); CREATE INDEX idx_container_exp_policies_on_project_id_next_run_at_enabled ON container_expiration_policies USING btree (project_id, next_run_at, enabled); @@ -23662,6 +23688,8 @@ CREATE INDEX idx_on_protected_branch ON approval_group_rules_protected_branches CREATE INDEX idx_open_issues_on_project_and_confidential_and_author_and_id ON issues USING btree (project_id, confidential, author_id, id) WHERE (state_id = 1); +CREATE INDEX idx_p_catalog_resource_component_usages_on_catalog_resource_id ON ONLY p_catalog_resource_component_usages USING btree (catalog_resource_id); + CREATE INDEX idx_packages_debian_group_component_files_on_architecture_id ON packages_debian_group_component_files USING btree (architecture_id); CREATE INDEX idx_packages_debian_project_component_files_on_architecture_id ON packages_debian_project_component_files USING btree (architecture_id); @@ -26024,6 +26052,8 @@ CREATE INDEX index_organizations_on_path_trigram ON organizations USING gin (pat CREATE UNIQUE INDEX index_organizations_on_unique_name_per_group ON customer_relations_organizations USING btree (group_id, lower(name), id); +CREATE INDEX index_p_catalog_resource_component_usages_on_project_id ON ONLY p_catalog_resource_component_usages USING btree (project_id); + CREATE INDEX index_p_catalog_resource_sync_events_on_id_where_pending ON ONLY p_catalog_resource_sync_events USING btree (id) WHERE (status = 1); CREATE INDEX index_p_ci_finished_build_ch_sync_events_finished_at ON ONLY p_ci_finished_build_ch_sync_events USING btree (partition, build_finished_at); @@ -30604,6 +30634,9 @@ ALTER TABLE ONLY operations_user_lists ALTER TABLE ONLY resource_link_events ADD CONSTRAINT fk_rails_0cea73eba5 FOREIGN KEY (child_work_item_id) REFERENCES issues(id) ON DELETE CASCADE; +ALTER TABLE p_catalog_resource_component_usages + ADD CONSTRAINT fk_rails_0e15a4677f FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY audit_events_google_cloud_logging_configurations ADD CONSTRAINT fk_rails_0eb52fc617 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE; @@ -31585,6 +31618,9 @@ ALTER TABLE ONLY alert_management_alert_assignees ALTER TABLE ONLY scim_identities ADD CONSTRAINT fk_rails_9421a0bffb FOREIGN KEY (user_id) REFERENCES users(id) ON DELETE CASCADE; +ALTER TABLE p_catalog_resource_component_usages + ADD CONSTRAINT fk_rails_9430673479 FOREIGN KEY (catalog_resource_id) REFERENCES catalog_resources(id) ON DELETE CASCADE; + ALTER TABLE ONLY packages_debian_project_distributions ADD CONSTRAINT fk_rails_94b95e1f84 FOREIGN KEY (creator_id) REFERENCES users(id) ON DELETE SET NULL; @@ -32131,6 +32167,9 @@ ALTER TABLE ONLY label_priorities ALTER TABLE ONLY packages_packages ADD CONSTRAINT fk_rails_e1ac527425 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE p_catalog_resource_component_usages + ADD CONSTRAINT fk_rails_e1ba64b7ee FOREIGN KEY (component_id) REFERENCES catalog_resource_components(id) ON DELETE CASCADE; + ALTER TABLE ONLY cluster_platforms_kubernetes ADD CONSTRAINT fk_rails_e1e2cf841a FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE; diff --git a/ee/spec/models/factories_spec.rb b/ee/spec/models/factories_spec.rb index 3b2910fb93d0cd..395c56f78a38ab 100644 --- a/ee/spec/models/factories_spec.rb +++ b/ee/spec/models/factories_spec.rb @@ -135,6 +135,7 @@ # is being mutated. skip_factory_defaults = %i[ ci_catalog_resource_component + ci_catalog_resource_component_usage ci_catalog_resource_version ci_job_token_project_scope_link ci_subscriptions_project diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index f7f570c43e6b9b..38f3d9695505a9 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -102,6 +102,7 @@ p_ci_builds: %w[erased_by_id trigger_request_id partition_id auto_canceled_by_partition_id], p_batched_git_ref_updates_deletions: %w[project_id partition_id], p_catalog_resource_sync_events: %w[catalog_resource_id project_id partition_id], + p_catalog_resource_component_usages: %w[used_by_project_id], # No FK constraint because we want to preserve historical usage data p_ci_finished_build_ch_sync_events: %w[build_id], p_ci_job_artifacts: %w[partition_id project_id job_id], p_ci_pipeline_variables: %w[partition_id], diff --git a/spec/factories/ci/catalog/resources/components/usages.rb b/spec/factories/ci/catalog/resources/components/usages.rb new file mode 100644 index 00000000000000..2fe358e90724fb --- /dev/null +++ b/spec/factories/ci/catalog/resources/components/usages.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :ci_catalog_resource_component_usage, class: 'Ci::Catalog::Resources::Components::Usage' do + component factory: :ci_catalog_resource_component + catalog_resource { component.catalog_resource } + project { component.project } + used_by_project_id { 1 } + end +end diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 9cc72612583691..9b49b829e5181b 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -533,6 +533,7 @@ project: - catalog_resource_sync_events - catalog_resource_versions - ci_components +- ci_component_usages - external_status_checks - base_tags - project_topics diff --git a/spec/models/ci/catalog/resource_spec.rb b/spec/models/ci/catalog/resource_spec.rb index 20349a02352d8d..1aa71c9473e710 100644 --- a/spec/models/ci/catalog/resource_spec.rb +++ b/spec/models/ci/catalog/resource_spec.rb @@ -26,6 +26,12 @@ have_many(:components).class_name('Ci::Catalog::Resources::Component').with_foreign_key(:catalog_resource_id)) end + it do + is_expected.to( + have_many(:component_usages).class_name('Ci::Catalog::Resources::Components::Usage') + .with_foreign_key(:catalog_resource_id)) + end + it do is_expected.to( have_many(:versions).class_name('Ci::Catalog::Resources::Version').with_foreign_key(:catalog_resource_id)) diff --git a/spec/models/ci/catalog/resources/component_spec.rb b/spec/models/ci/catalog/resources/component_spec.rb index 2ee911759203e5..bf0506f61658c5 100644 --- a/spec/models/ci/catalog/resources/component_spec.rb +++ b/spec/models/ci/catalog/resources/component_spec.rb @@ -8,6 +8,7 @@ it { is_expected.to belong_to(:catalog_resource).class_name('Ci::Catalog::Resource') } it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:version).class_name('Ci::Catalog::Resources::Version') } + it { is_expected.to have_many(:usages).class_name('Ci::Catalog::Resources::Components::Usage') } it_behaves_like 'a BulkInsertSafe model', described_class do let_it_be(:project) { create(:project, :readme, description: 'project description') } diff --git a/spec/models/ci/catalog/resources/components/usage_spec.rb b/spec/models/ci/catalog/resources/components/usage_spec.rb new file mode 100644 index 00000000000000..c8d74bef7c5920 --- /dev/null +++ b/spec/models/ci/catalog/resources/components/usage_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::Catalog::Resources::Components::Usage, type: :model, feature_category: :pipeline_composition do + let_it_be(:component) { create(:ci_catalog_resource_component) } + let(:component_usage) { build(:ci_catalog_resource_component_usage, component: component) } + + it { is_expected.to belong_to(:component).class_name('Ci::Catalog::Resources::Component') } + it { is_expected.to belong_to(:catalog_resource).class_name('Ci::Catalog::Resource') } + it { is_expected.to belong_to(:project).class_name('Project') } + + describe 'validations' do + it { is_expected.to validate_presence_of(:component) } + it { is_expected.to validate_presence_of(:catalog_resource) } + it { is_expected.to validate_presence_of(:project) } + it { is_expected.to validate_presence_of(:used_by_project_id) } + + it do + component_usage.save! + + expect(component_usage).to validate_uniqueness_of(:used_date) + .scoped_to([:component_id, :used_by_project_id]) + end + end + + describe 'callbacks' do + describe 'used date', :freeze_time do + context 'when used date is not provided' do + it 'sets the used date to today' do + component_usage.save! + + expect(component_usage.reload.used_date).to eq(Date.today) + end + end + + context 'when used date is provided' do + it 'sets the given used date' do + component_usage.used_date = Date.today + 1.day + component_usage.save! + + expect(component_usage.reload.used_date).to eq(Date.today + 1.day) + end + end + end + end + + describe 'monthly partitioning', :freeze_time do + let(:partition_manager) { Gitlab::Database::Partitioning::PartitionManager.new(described_class) } + + it 'drops partitions older than 12 months' do + # We start with the intialized partitions + oldest_partition = described_class.partitioning_strategy.current_partitions.min_by(&:from) + newest_partition = described_class.partitioning_strategy.current_partitions.max_by(&:from) + + # We add one usage record into the oldest and newest partitions + create(:ci_catalog_resource_component_usage, component: component, used_date: oldest_partition.from) + create(:ci_catalog_resource_component_usage, component: component, used_date: newest_partition.from) + + expect(described_class.count).to eq(2) + + # After traveling forward 12 months from the oldest partition month + travel_to(oldest_partition.to + 12.months + 1.day) + + # the oldest partition is dropped + partition_manager.sync_partitions + + expect(described_class.partitioning_strategy.current_partitions.include?(oldest_partition)).to eq(false) + + # and we only have the usage record from the remaining partitions + expect(described_class.count).to eq(1) + end + end +end diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 36e715985de19a..8bc5aff6e02941 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -49,6 +49,7 @@ it { is_expected.to have_one(:slack_integration) } it { is_expected.to have_one(:catalog_resource) } it { is_expected.to have_many(:ci_components).class_name('Ci::Catalog::Resources::Component') } + it { is_expected.to have_many(:ci_component_usages).class_name('Ci::Catalog::Resources::Components::Usage') } it { is_expected.to have_many(:catalog_resource_versions).class_name('Ci::Catalog::Resources::Version') } it { is_expected.to have_many(:catalog_resource_sync_events).class_name('Ci::Catalog::Resources::SyncEvent') } it { is_expected.to have_one(:microsoft_teams_integration) } -- GitLab From ee02fb61e15e6f95ab4bd4b65f62ebf6e0ebbe05 Mon Sep 17 00:00:00 2001 From: Leaminn Ma Date: Wed, 6 Mar 2024 17:11:15 +0000 Subject: [PATCH 2/3] Add clarifying comment to ci_component_usages relationship in Project --- app/models/project.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/project.rb b/app/models/project.rb index 978c2520df4191..738a5210a78e1b 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -189,6 +189,7 @@ class Project < ApplicationRecord has_one :catalog_resource, class_name: 'Ci::Catalog::Resource', inverse_of: :project has_many :ci_components, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :project + # These are usages of the ci_components owned (not used) by the project has_many :ci_component_usages, class_name: 'Ci::Catalog::Resources::Components::Usage', inverse_of: :project has_many :catalog_resource_versions, class_name: 'Ci::Catalog::Resources::Version', inverse_of: :project has_many :catalog_resource_sync_events, class_name: 'Ci::Catalog::Resources::SyncEvent', inverse_of: :project -- GitLab From e1adb8c951ff9f59cbb43dd242ca5166c15e69ef Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Thu, 7 Mar 2024 18:42:35 +0000 Subject: [PATCH 3/3] Update TO DO comment to TODO --- app/models/ci/catalog/resources/components/usage.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/ci/catalog/resources/components/usage.rb b/app/models/ci/catalog/resources/components/usage.rb index 119e61ab11bbfa..b721e57d0a75ea 100644 --- a/app/models/ci/catalog/resources/components/usage.rb +++ b/app/models/ci/catalog/resources/components/usage.rb @@ -13,7 +13,7 @@ class Usage < ::ApplicationRecord self.table_name = 'p_catalog_resource_component_usages' self.primary_key = :id - # TO DO: Retention period to be shortened in https://gitlab.com/gitlab-org/gitlab/-/issues/443681 + # TODO: Retention period to be shortened in https://gitlab.com/gitlab-org/gitlab/-/issues/443681 partitioned_by :used_date, strategy: :monthly, retain_for: 12.months belongs_to :component, class_name: 'Ci::Catalog::Resources::Component', inverse_of: :usages -- GitLab