diff --git a/app/events/projects/release_published_event.rb b/app/events/projects/release_published_event.rb new file mode 100644 index 0000000000000000000000000000000000000000..f0be95b893ed804c27362b5162e75d1cc430c180 --- /dev/null +++ b/app/events/projects/release_published_event.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Projects + class ReleasePublishedEvent < ::Gitlab::EventStore::Event + def schema + { + 'type' => 'object', + 'properties' => { + 'release_id' => { 'type' => 'integer' } + }, + 'required' => %w[release_id] + } + end + end +end diff --git a/app/models/release.rb b/app/models/release.rb index 1cd623e1254a440e744552709fd676d0d4763a92..7bacc69f038239bed23109a298a365db6b9efb42 100644 --- a/app/models/release.rb +++ b/app/models/release.rb @@ -54,6 +54,7 @@ class Release < ApplicationRecord scope :recent, -> { sorted.limit(MAX_NUMBER_TO_DISPLAY) } scope :without_evidence, -> { left_joins(:evidences).where(::Releases::Evidence.arel_table[:id].eq(nil)) } scope :released_within_2hrs, -> { where(released_at: Time.zone.now - 1.hour..Time.zone.now + 1.hour) } + scope :unpublished, -> { where(release_published_at: nil) } scope :for_projects, ->(projects) { where(project_id: projects) } scope :by_tag, ->(tag) { where(tag: tag) } @@ -66,6 +67,7 @@ class Release < ApplicationRecord delegate :repository, to: :project MAX_NUMBER_TO_DISPLAY = 3 + MAX_NUMBER_TO_PUBLISH = 5000 class << self # In the future, we should support `order_by=semver`; @@ -97,6 +99,10 @@ def latest_for_projects(projects, order_by: 'released_at') .from("(VALUES #{project_ids_list}) projects (id)") .joins("INNER JOIN LATERAL (#{join_query.to_sql}) #{Release.table_name} ON TRUE") end + + def waiting_for_publish_event + unpublished.released_within_2hrs.joins(:project).merge(Project.with_feature_enabled(:releases)).limit(MAX_NUMBER_TO_PUBLISH) + end end def to_param diff --git a/app/workers/all_queues.yml b/app/workers/all_queues.yml index ec5156bb1d0de3f88978d50ede0b769928f12d98..90198e99433695034e7aff96305c9233138c214b 100644 --- a/app/workers/all_queues.yml +++ b/app/workers/all_queues.yml @@ -786,6 +786,15 @@ :weight: 1 :idempotent: false :tags: [] +- :name: cronjob:releases_publish_event + :worker_name: Releases::PublishEventWorker + :feature_category: :release_orchestration + :has_external_dependencies: false + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: cronjob:remove_expired_group_links :worker_name: RemoveExpiredGroupLinksWorker :feature_category: :system_access diff --git a/app/workers/releases/publish_event_worker.rb b/app/workers/releases/publish_event_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..8bcc580dceb6c4892411d9dc84bcf7bc482a85c5 --- /dev/null +++ b/app/workers/releases/publish_event_worker.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +module Releases + class PublishEventWorker + include ApplicationWorker + include CronjobQueue + + idempotent! + data_consistency :always # rubocop:disable SidekiqLoadBalancing/WorkerDataConsistency -- usual for EventStore jobs. + feature_category :release_orchestration + + def perform + releases_published = 0 + + Release.waiting_for_publish_event.each_batch(of: 100) do |releases| + releases.each do |release| + with_context(project: release.project) do + ::Gitlab::EventStore.publish( + ::Projects::ReleasePublishedEvent.new(data: { release_id: release.id }) + ) + + releases_published += 1 + end + end + + releases.touch_all(:release_published_at) + end + + log_extra_metadata_on_done(:releases_published, releases_published) if releases_published > 0 + end + end +end diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index ade5465f8ea7ab8da1927e0a839adcf896cca2cd..cd2bfd85d0a303901d7484566476918923737ca3 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -634,6 +634,9 @@ Settings.cron_jobs['manage_evidence_worker'] ||= {} Settings.cron_jobs['manage_evidence_worker']['cron'] ||= '0 * * * *' Settings.cron_jobs['manage_evidence_worker']['job_class'] = 'Releases::ManageEvidenceWorker' +Settings.cron_jobs['publish_release_worker'] ||= {} +Settings.cron_jobs['publish_release_worker']['cron'] ||= '20,50 * * * *' +Settings.cron_jobs['publish_release_worker']['job_class'] = 'Releases::PublishEventWorker' Settings.cron_jobs['user_status_cleanup_batch_worker'] ||= {} Settings.cron_jobs['user_status_cleanup_batch_worker']['cron'] ||= '* * * * *' Settings.cron_jobs['user_status_cleanup_batch_worker']['job_class'] = 'UserStatusCleanup::BatchWorker' diff --git a/db/migrate/20231212154022_add_release_published_at_to_release.rb b/db/migrate/20231212154022_add_release_published_at_to_release.rb new file mode 100644 index 0000000000000000000000000000000000000000..8ecb51a8cf12b1b71b20398de38f23a23e2d2e28 --- /dev/null +++ b/db/migrate/20231212154022_add_release_published_at_to_release.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class AddReleasePublishedAtToRelease < Gitlab::Database::Migration[2.2] + milestone '16.8' + enable_lock_retries! + + def change + add_column :releases, :release_published_at, :datetime_with_timezone + end +end diff --git a/db/migrate/20231220094609_add_release_published_at_index_to_release.rb b/db/migrate/20231220094609_add_release_published_at_index_to_release.rb new file mode 100644 index 0000000000000000000000000000000000000000..bc5891ad1983ab5359d796fe8d778cf2ee2932cc --- /dev/null +++ b/db/migrate/20231220094609_add_release_published_at_index_to_release.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +class AddReleasePublishedAtIndexToRelease < Gitlab::Database::Migration[2.2] + milestone '16.8' + disable_ddl_transaction! + + def up + add_concurrent_index :releases, :release_published_at, name: 'releases_published_at_index' + end + + def down + remove_concurrent_index :releases, :release_published_at, name: 'releases_published_at_index' + end +end diff --git a/db/schema_migrations/20231212154022 b/db/schema_migrations/20231212154022 new file mode 100644 index 0000000000000000000000000000000000000000..9f9967d03263456dba22bb1afef42d1468ad9d42 --- /dev/null +++ b/db/schema_migrations/20231212154022 @@ -0,0 +1 @@ +c005eb8901f1ebb85dedb044d627396f591bd760a0315dc3f45171def0f972e5 \ No newline at end of file diff --git a/db/schema_migrations/20231220094609 b/db/schema_migrations/20231220094609 new file mode 100644 index 0000000000000000000000000000000000000000..30ece81d4a013b110cb1309557608ddb2c51be08 --- /dev/null +++ b/db/schema_migrations/20231220094609 @@ -0,0 +1 @@ +aab891f39866b4933cadd8295ecaa1c9f8a256cda832b734dfb1911580187bf3 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 754c8139b3726944e631d7d2f09f72b803436d80..e3900b7b7ebf2231a298739fde00bb312da706ef 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -22754,7 +22754,8 @@ CREATE TABLE releases ( author_id integer, name character varying, sha character varying, - released_at timestamp with time zone NOT NULL + released_at timestamp with time zone NOT NULL, + release_published_at timestamp with time zone ); CREATE SEQUENCE releases_id_seq @@ -35395,6 +35396,8 @@ CREATE INDEX partial_index_user_id_app_id_created_at_token_not_revoked ON oauth_ CREATE UNIQUE INDEX pm_checkpoints_path_components ON pm_checkpoints USING btree (purl_type, data_type, version_format); +CREATE INDEX releases_published_at_index ON releases USING btree (release_published_at); + CREATE INDEX scan_finding_approval_mr_rule_index_id ON approval_merge_request_rules USING btree (id) WHERE (report_type = 4); CREATE INDEX scan_finding_approval_mr_rule_index_merge_request_id ON approval_merge_request_rules USING btree (merge_request_id) WHERE (report_type = 4); diff --git a/spec/models/release_spec.rb b/spec/models/release_spec.rb index bff9f73e44aaeefa9ed12d4259a324d4290f709a..4a4cb1ae46a4f7c572081fa503bb7f9a60d8e146 100644 --- a/spec/models/release_spec.rb +++ b/spec/models/release_spec.rb @@ -156,6 +156,7 @@ describe 'latest releases' do let_it_be(:yesterday) { Time.zone.now - 1.day } + let_it_be(:today) { Time.zone.now } let_it_be(:tomorrow) { Time.zone.now + 1.day } let_it_be(:project2) { create(:project) } @@ -176,6 +177,14 @@ create(:release, project: project2, released_at: tomorrow, created_at: yesterday) end + let_it_be(:project2_release3) do + create(:release, project: project2, released_at: today, created_at: yesterday) + end + + let_it_be(:project2_release4) do + create(:release, project: project2, released_at: today, created_at: yesterday, release_published_at: today) + end + let(:args) { {} } describe '.latest' do @@ -240,6 +249,16 @@ end end end + + describe '.waiting_for_publish_event' do + let(:releases) { [project2_release3] } + + subject(:waiting) { described_class.waiting_for_publish_event } + + it "find today's releases not yet published" do + expect(waiting).to match_array(releases) + end + end end describe '#assets_count' do diff --git a/spec/workers/releases/publish_event_worker_spec.rb b/spec/workers/releases/publish_event_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..86dd09a756fc469bcf05227ab71450ab47312436 --- /dev/null +++ b/spec/workers/releases/publish_event_worker_spec.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Releases::PublishEventWorker, feature_category: :release_evidence do + let_it_be(:project) { create(:project, :repository) } + let_it_be_with_reload(:release) { create(:release, project: project, released_at: Time.current) } + + before do + allow(Gitlab::EventStore).to receive(:publish).and_return(true) + end + + describe 'when the releases feature is not disabled' do + before do + project.update!(releases_access_level: 'enabled') + described_class.new.perform + end + + it 'broadcasts the published event' do + expect(Gitlab::EventStore).to have_received(:publish).with(Projects::ReleasePublishedEvent) + end + + it 'sets the release as published' do + expect(release.release_published_at).not_to be_nil + end + end + + describe 'when the releases feature is disabled' do + before do + project.update!(releases_access_level: 'disabled') + described_class.new.perform + end + + it 'does not broadcasts the published event' do + expect(Gitlab::EventStore).not_to have_received(:publish).with(Projects::ReleasePublishedEvent) + end + + # Having a release created with the releases feature disabled is a bogus state anyway. + # Setting it as published prevents having such releases piling up forever in the + # `unpublished` scope. + it 'sets the release as published' do + expect(release.release_published_at).not_to be_nil + end + end +end