diff --git a/app/models/projects/data_transfer.rb b/app/models/projects/data_transfer.rb new file mode 100644 index 0000000000000000000000000000000000000000..a93aea557810b429f3c3498a7fdf79fa27807d31 --- /dev/null +++ b/app/models/projects/data_transfer.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# Tracks egress of various services per project +# This class ensures that we keep 1 record per project per month. +module Projects + class DataTransfer < ApplicationRecord + self.table_name = 'project_data_transfers' + + belongs_to :project + belongs_to :namespace + + scope :current_month, -> { where(date: beginning_of_month) } + + def self.beginning_of_month(time = Time.current) + time.utc.beginning_of_month + end + end +end diff --git a/db/docs/project_data_transfers.yml b/db/docs/project_data_transfers.yml new file mode 100644 index 0000000000000000000000000000000000000000..cf06c61cad6e37d80bf46dea4b95b6319f070cb5 --- /dev/null +++ b/db/docs/project_data_transfers.yml @@ -0,0 +1,10 @@ +--- +table_name: project_data_transfers +classes: +- Projects::DataTransfer +feature_categories: +- source_code_management +description: Data transfer metrics per project +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/107970 +milestone: '15.9' +gitlab_schema: gitlab_main diff --git a/db/migrate/20230117173433_create_project_data_transfer.rb b/db/migrate/20230117173433_create_project_data_transfer.rb new file mode 100644 index 0000000000000000000000000000000000000000..f63191c991b957eb2eb070c4622d3b5ac360f27d --- /dev/null +++ b/db/migrate/20230117173433_create_project_data_transfer.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class CreateProjectDataTransfer < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + def up + with_lock_retries do + create_table :project_data_transfers do |t| + t.references :project, index: false, null: false + t.references :namespace, index: true, null: false + t.bigint :repository_egress, null: false, default: 0 + t.bigint :artifacts_egress, null: false, default: 0 + t.bigint :packages_egress, null: false, default: 0 + t.bigint :registry_egress, null: false, default: 0 + t.date :date, null: false + t.datetime_with_timezone :created_at, null: false + + t.index [:project_id, :namespace_id, :date], unique: true, + name: 'index_project_data_transfers_on_project_and_namespace_and_date' + end + end + + add_check_constraint :project_data_transfers, + "(date = date_trunc('month', date))", 'project_data_transfers_project_year_month_constraint' + end + + def down + with_lock_retries do + drop_table :project_data_transfers + end + end +end diff --git a/db/schema_migrations/20230117173433 b/db/schema_migrations/20230117173433 new file mode 100644 index 0000000000000000000000000000000000000000..cef427805470ef2cb850a5347110782338f5767d --- /dev/null +++ b/db/schema_migrations/20230117173433 @@ -0,0 +1 @@ +ee7f3ba064eaaf4a1bf92e5c0a2ed32e5d294ddd6f1fdd8e6eed54c8b83c2af5 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 8e1a19b3940aae76b89de08e3374cb5170c59788..41785ae6263804ed4d180ccbe6d83c44b79d8c00 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -20156,6 +20156,28 @@ CREATE SEQUENCE project_daily_statistics_id_seq ALTER SEQUENCE project_daily_statistics_id_seq OWNED BY project_daily_statistics.id; +CREATE TABLE project_data_transfers ( + id bigint NOT NULL, + project_id bigint NOT NULL, + namespace_id bigint NOT NULL, + repository_egress bigint DEFAULT 0 NOT NULL, + artifacts_egress bigint DEFAULT 0 NOT NULL, + packages_egress bigint DEFAULT 0 NOT NULL, + registry_egress bigint DEFAULT 0 NOT NULL, + date date NOT NULL, + created_at timestamp with time zone NOT NULL, + CONSTRAINT project_data_transfers_project_year_month_constraint CHECK ((date = date_trunc('month'::text, (date)::timestamp with time zone))) +); + +CREATE SEQUENCE project_data_transfers_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE project_data_transfers_id_seq OWNED BY project_data_transfers.id; + CREATE TABLE project_deploy_tokens ( id integer NOT NULL, project_id integer NOT NULL, @@ -24507,6 +24529,8 @@ ALTER TABLE ONLY project_custom_attributes ALTER COLUMN id SET DEFAULT nextval(' ALTER TABLE ONLY project_daily_statistics ALTER COLUMN id SET DEFAULT nextval('project_daily_statistics_id_seq'::regclass); +ALTER TABLE ONLY project_data_transfers ALTER COLUMN id SET DEFAULT nextval('project_data_transfers_id_seq'::regclass); + ALTER TABLE ONLY project_deploy_tokens ALTER COLUMN id SET DEFAULT nextval('project_deploy_tokens_id_seq'::regclass); ALTER TABLE ONLY project_export_jobs ALTER COLUMN id SET DEFAULT nextval('project_export_jobs_id_seq'::regclass); @@ -26733,6 +26757,9 @@ ALTER TABLE ONLY project_custom_attributes ALTER TABLE ONLY project_daily_statistics ADD CONSTRAINT project_daily_statistics_pkey PRIMARY KEY (id); +ALTER TABLE ONLY project_data_transfers + ADD CONSTRAINT project_data_transfers_pkey PRIMARY KEY (id); + ALTER TABLE ONLY project_deploy_tokens ADD CONSTRAINT project_deploy_tokens_pkey PRIMARY KEY (id); @@ -30611,6 +30638,10 @@ CREATE UNIQUE INDEX index_project_custom_attributes_on_project_id_and_key ON pro CREATE UNIQUE INDEX index_project_daily_statistics_on_project_id_and_date ON project_daily_statistics USING btree (project_id, date DESC); +CREATE INDEX index_project_data_transfers_on_namespace_id ON project_data_transfers USING btree (namespace_id); + +CREATE UNIQUE INDEX index_project_data_transfers_on_project_and_namespace_and_date ON project_data_transfers USING btree (project_id, namespace_id, date); + CREATE INDEX index_project_deploy_tokens_on_deploy_token_id ON project_deploy_tokens USING btree (deploy_token_id); CREATE UNIQUE INDEX index_project_deploy_tokens_on_project_id_and_deploy_token_id ON project_deploy_tokens USING btree (project_id, deploy_token_id); diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 4f28017c16bda3a72813f0fcb2cdd54e8ac87fd8..6b3b1613cc02a124e8f360de54034ac81ddc0bd6 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -89,6 +89,7 @@ oauth_applications: %w[owner_id], product_analytics_events_experimental: %w[event_id txn_id user_id], project_build_artifacts_size_refreshes: %w[last_job_artifact_id], + project_data_transfers: %w[project_id namespace_id], project_error_tracking_settings: %w[sentry_project_id], project_group_links: %w[group_id], project_statistics: %w[namespace_id], diff --git a/spec/factories/projects/data_transfers.rb b/spec/factories/projects/data_transfers.rb new file mode 100644 index 0000000000000000000000000000000000000000..4184f4756634cebbab5819fb1801c6e8707cf77b --- /dev/null +++ b/spec/factories/projects/data_transfers.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :project_data_transfer, class: 'Projects::DataTransfer' do + project factory: :project + namespace { project.root_namespace } + date { Time.current.utc.beginning_of_month } + end +end diff --git a/spec/models/projects/data_transfer_spec.rb b/spec/models/projects/data_transfer_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..6d3ddbdd74efc476003015a5e9e9fa6208b70e45 --- /dev/null +++ b/spec/models/projects/data_transfer_spec.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::DataTransfer, feature_category: :source_code_management do + let_it_be(:project) { create(:project) } + + it { expect(subject).to be_valid } + + describe 'associations' do + it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:namespace) } + end + + describe 'scopes' do + describe '.current_month' do + subject { described_class.current_month } + + it 'returns data transfer for the current month' do + travel_to(Time.utc(2022, 5, 2)) do + _past_month = create(:project_data_transfer, project: project, date: '2022-04-01') + current_month = create(:project_data_transfer, project: project, date: '2022-05-01') + + is_expected.to match_array([current_month]) + end + end + end + end + + describe '.beginning_of_month' do + subject { described_class.beginning_of_month(time) } + + let(:time) { Time.utc(2022, 5, 2) } + + it { is_expected.to eq(Time.utc(2022, 5, 1)) } + end + + describe 'unique index' do + before do + create(:project_data_transfer, project: project, date: '2022-05-01') + end + + it 'raises unique index violation' do + expect { create(:project_data_transfer, project: project, namespace: project.root_namespace, date: '2022-05-01') } + .to raise_error(ActiveRecord::RecordNotUnique) + end + + context 'when project was moved from one namespace to another' do + it 'creates a new record' do + expect { create(:project_data_transfer, project: project, namespace: create(:namespace), date: '2022-05-01') } + .to change { described_class.count }.by(1) + end + end + + context 'when a different project is created' do + it 'creates a new record' do + expect { create(:project_data_transfer, project: build(:project), date: '2022-05-01') } + .to change { described_class.count }.by(1) + end + end + end +end