diff --git a/doc/api/graphql/reference/_index.md b/doc/api/graphql/reference/_index.md index 545234f0c423a586a7030b6a1042efa724b6a3e7..7fa8ef49c4276ad9c2c7b151a0b1cd008b3b6e28 100644 --- a/doc/api/graphql/reference/_index.md +++ b/doc/api/graphql/reference/_index.md @@ -1667,6 +1667,23 @@ four standard [pagination arguments](#pagination-arguments): | ---- | ---- | ----------- | | `projectPath` | [`ID!`](#id) | Project the secret permission belong to. | +### `Query.securityPoliciesSyncStatus` + +{{< details >}} +**Introduced** in GitLab 18.4. +**Status**: Experiment. +{{< /details >}} + +Get the current security policy synchronization status. + +Returns [`PoliciesSyncUpdated`](#policiessyncupdated). + +#### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `policyConfigurationId` | [`SecurityOrchestrationPolicyConfigurationID!`](#securityorchestrationpolicyconfigurationid) | ID of the security orchestration policy configuration. | + ### `Query.selfManagedAddOnEligibleUsers` {{< details >}} @@ -38294,6 +38311,7 @@ Security policy state synchronization update. Returns `null` if the `security_po | Name | Type | Description | | ---- | ---- | ----------- | | `failedProjects` | [`[String!]`](#string) | IDs of failed projects. | +| `inProgress` | [`Boolean`](#boolean) | Whether security policies are currently being synchronized. | | `mergeRequestsProgress` | [`Float`](#float) | Percentage of merge requests synced. | | `mergeRequestsTotal` | [`Int`](#int) | Total number of merge requests synced. | | `projectsProgress` | [`Float`](#float) | Percentage of projects synced. | diff --git a/ee/app/graphql/ee/graphql_triggers.rb b/ee/app/graphql/ee/graphql_triggers.rb index 9269de6a64451d026d342fb200173422f4212e24..4543b8aa6c7968b606cae2fb1ef8f1f36c6f0341 100644 --- a/ee/app/graphql/ee/graphql_triggers.rb +++ b/ee/app/graphql/ee/graphql_triggers.rb @@ -75,13 +75,15 @@ def self.security_policies_sync_updated( projects_total, failed_projects, merge_requests_progress, - merge_requests_total + merge_requests_total, + in_progress ) ::GitlabSchema.subscriptions.trigger( :security_policies_sync_updated, { policy_configuration_id: policy_configuration.to_global_id }, { projects_progress: projects_progress, projects_total: projects_total, failed_projects: failed_projects, - merge_requests_progress: merge_requests_progress, merge_requests_total: merge_requests_total } + merge_requests_progress: merge_requests_progress, merge_requests_total: merge_requests_total, + in_progress: in_progress } ) end end diff --git a/ee/app/graphql/ee/types/query_type.rb b/ee/app/graphql/ee/types/query_type.rb index 862f8b3f9d119be41707a0afd3b53efda00441ba..3c97138fda820f8913023a6da0ab30ac1355c6dd 100644 --- a/ee/app/graphql/ee/types/query_type.rb +++ b/ee/app/graphql/ee/types/query_type.rb @@ -418,6 +418,12 @@ module QueryType description: 'Allowed work item statuses from the root groups the current user belongs to.', experiment: { milestone: '18.4' }, resolver: ::Resolvers::WorkItems::AllowedStatusesResolver + + field :security_policies_sync_status, ::Types::GitlabSubscriptions::Security::PoliciesSyncUpdated, + null: true, + resolver: ::Resolvers::SecurityOrchestration::PoliciesSyncStatusResolver, + description: 'Get the current security policy synchronization status.', + experiment: { milestone: '18.4' } end def vulnerability(id:) diff --git a/ee/app/graphql/resolvers/security_orchestration/policies_sync_status_resolver.rb b/ee/app/graphql/resolvers/security_orchestration/policies_sync_status_resolver.rb new file mode 100644 index 0000000000000000000000000000000000000000..c686acb70fd3fb1974c4c5b4e226daa05dfa9f19 --- /dev/null +++ b/ee/app/graphql/resolvers/security_orchestration/policies_sync_status_resolver.rb @@ -0,0 +1,39 @@ +# frozen_string_literal: true + +module Resolvers + module SecurityOrchestration # rubocop:disable Gitlab/BoundedContexts -- TODO: Namespacing + class PoliciesSyncStatusResolver < BaseResolver + type ::Types::GitlabSubscriptions::Security::PoliciesSyncUpdated, null: true + + argument :policy_configuration_id, ::Types::GlobalIDType[::Security::OrchestrationPolicyConfiguration], + required: true, + description: 'ID of the security orchestration policy configuration.' + + def authorized?(policy_configuration_id:) + policy_configuration = find_policy_configuration(policy_configuration_id) + + return false unless policy_configuration + + policy_project = policy_configuration.security_policy_management_project + + return false unless current_user.can?(:update_security_orchestration_policy_project, policy_project) + + Feature.enabled?(:security_policy_sync_propagation_tracking, policy_project) + end + + def resolve(policy_configuration_id:) + policy_configuration = find_policy_configuration(policy_configuration_id) + + return {} unless policy_configuration + + ::Security::SecurityOrchestrationPolicies::PolicySyncState::State.new(policy_configuration.id).to_h + end + + private + + def find_policy_configuration(id) + ::Gitlab::Graphql::Lazy.force(GitlabSchema.find_by_gid(id)) + end + end + end +end diff --git a/ee/app/graphql/subscriptions/security/policies_sync_updated.rb b/ee/app/graphql/subscriptions/security/policies_sync_updated.rb index 37ca81ef9318ae181f3555bfd92220e9ffc81cd4..ff571addd417a040466ebfe467a4f3f59567e8c0 100644 --- a/ee/app/graphql/subscriptions/security/policies_sync_updated.rb +++ b/ee/app/graphql/subscriptions/security/policies_sync_updated.rb @@ -28,7 +28,8 @@ def update(_) projects_total: object[:projects_total], failed_projects: object[:failed_projects], merge_requests_progress: object[:merge_requests_progress], - merge_requests_total: object[:merge_requests_total] + merge_requests_total: object[:merge_requests_total], + in_progress: object[:in_progress] } end diff --git a/ee/app/graphql/types/gitlab_subscriptions/security/policies_sync_updated.rb b/ee/app/graphql/types/gitlab_subscriptions/security/policies_sync_updated.rb index e297921bedffd9ee0e14057236275ff52f44ce07..4bf53f7a143630e6a921221e3538aa7016376eb8 100644 --- a/ee/app/graphql/types/gitlab_subscriptions/security/policies_sync_updated.rb +++ b/ee/app/graphql/types/gitlab_subscriptions/security/policies_sync_updated.rb @@ -30,6 +30,10 @@ class PoliciesSyncUpdated < ::Types::BaseObject field :merge_requests_total, GraphQL::Types::Int, null: true, description: 'Total number of merge requests synced.' + + field :in_progress, GraphQL::Types::Boolean, + null: true, + description: 'Whether security policies are currently being synchronized.' end # rubocop:enable GraphQL/ExtractType # rubocop:enable Graphql/AuthorizeTypes diff --git a/ee/lib/security/security_orchestration_policies/policy_sync_state.rb b/ee/lib/security/security_orchestration_policies/policy_sync_state.rb index 7baef6578fbe6caad8eacc208d86e564910bca87..ea2795d78b2c7b4ba325ab4267e06b013531af63 100644 --- a/ee/lib/security/security_orchestration_policies/policy_sync_state.rb +++ b/ee/lib/security/security_orchestration_policies/policy_sync_state.rb @@ -5,6 +5,7 @@ module SecurityOrchestrationPolicies module PolicySyncState POLICY_SYNC_TTL = 24.hours.to_i POLICY_SYNC_CONTEXT_KEY = :policy_sync_config_id + PROGRESS_MARKER = "" class State include Gitlab::Utils::StrongMemoize @@ -21,12 +22,51 @@ def initialize(config_id) @config_id = config_id end + def to_h + with_redis do |redis| + projects_pending = redis.scard(projects_sync_key) + projects_total = redis.get(total_projects_key).to_i + + merge_requests_pending = redis.scard(merge_requests_sync_key) + merge_requests_total = redis.get(total_merge_requests_key).to_i + + { + projects_progress: get_progress(projects_pending, projects_total), + projects_total: projects_total, + failed_projects: redis.smembers(failed_projects_sync_key), + merge_requests_progress: get_progress(merge_requests_pending, merge_requests_total), + merge_requests_total: merge_requests_total, + in_progress: sync_in_progress?(redis) + } + end + end + + # Mark as in progress + def start_sync + return if feature_disabled? + + with_redis do |redis| + redis.set(sync_in_progress_key, PROGRESS_MARKER, ex: POLICY_SYNC_TTL) + end + end + + # Mark sync as completed + def finish_sync + return if feature_disabled? + + with_redis do |redis| + redis.del(sync_in_progress_key) + end + end + # Appends project IDs, adding to the pending set and incrementing the total counter def append_projects(project_ids) return if feature_disabled? || project_ids.empty? with_redis do |redis| redis.multi do |multi| + multi.set(sync_in_progress_key, PROGRESS_MARKER, ex: POLICY_SYNC_TTL) + multi.sadd(projects_sync_key, project_ids) multi.incrby(total_projects_key, project_ids.size) @@ -70,6 +110,8 @@ def start_merge_request(merge_request_id) with_redis do |redis| redis.multi do |multi| + multi.set(sync_in_progress_key, PROGRESS_MARKER, ex: POLICY_SYNC_TTL) + multi.sadd(merge_requests_sync_key, merge_request_id) multi.incr(total_merge_requests_key) multi.set(merge_request_workers_sync_key(merge_request_id), 0) @@ -77,6 +119,7 @@ def start_merge_request(merge_request_id) multi.expire(merge_requests_sync_key, POLICY_SYNC_TTL) multi.expire(total_merge_requests_key, POLICY_SYNC_TTL) multi.expire(merge_request_workers_sync_key(merge_request_id), POLICY_SYNC_TTL) + multi.expire(sync_in_progress_key, POLICY_SYNC_TTL) end end end @@ -106,38 +149,27 @@ def finish_merge_request_worker(merge_request_id) end end - def sync_in_progress? + def sync_in_progress?(redis) return false if feature_disabled? - with_redis do |redis| + with_redis(redis) do |redis| conditions = redis.multi do |multi| - # rubocop:disable CodeReuse/ActiveRecord -- false positive - multi.exists?(total_projects_key) - multi.exists?(total_merge_requests_key) - # rubocop:enable CodeReuse/ActiveRecord - + multi.exists?(sync_in_progress_key) # rubocop:disable CodeReuse/ActiveRecord -- false positive multi.scard(projects_sync_key) multi.scard(merge_requests_sync_key) end - # rubocop:disable Layout/LineLength -- TODO - conditions.then do |total_projects, total_merge_requests, project_pending_count, merge_request_pending_count| - total_projects && total_merge_requests && (project_pending_count > 0 || merge_request_pending_count > 0) + conditions.then do |sync_in_progress, project_pending_count, merge_request_pending_count| + sync_in_progress || project_pending_count > 0 || merge_request_pending_count > 0 end - # rubocop:enable Layout/LineLength end end def clear return if feature_disabled? - with_redis do |redis| - redis.del(projects_sync_key) - redis.del(total_projects_key) - redis.del(failed_projects_sync_key) - redis.del(merge_requests_sync_key) - redis.del(total_merge_requests_key) - end + finish_sync + clear_pending_items end # Pending project IDs. @@ -182,6 +214,11 @@ def redis_key_tag "{security_policy_sync:#{config_id}}" end + # String: is sync currently in progress + def sync_in_progress_key + "#{redis_key_tag}:in_progress" + end + # Set: project IDs pending synchronization def projects_sync_key "#{redis_key_tag}:projects" @@ -212,25 +249,45 @@ def failed_projects_sync_key "#{redis_key_tag}:failed_projects" end + # Clear only pending items while keeping totals for historical data + def clear_pending_items + with_redis do |redis| + redis.multi do |multi| + multi.del(projects_sync_key) + multi.del(merge_requests_sync_key) + end + end + end + def get_progress(pending, total) - return if total == 0 + return 0.0 if total == 0 ((total - pending).to_f / total * 100).round end def trigger_subscription - projects_pending, projects_total, all_failed_projects, merge_requests_pending, merge_requests_total = + projects_pending, + projects_total, + all_failed_projects, + merge_requests_pending, + merge_requests_total, + in_progress = with_redis do |redis| [ redis.scard(projects_sync_key), redis.get(total_projects_key).to_i, redis.smembers(failed_projects_sync_key), redis.scard(merge_requests_sync_key), - redis.get(total_merge_requests_key).to_i + redis.get(total_merge_requests_key).to_i, + sync_in_progress?(redis) ] end - return unless sync_in_progress? + if projects_pending == 0 && merge_requests_pending == 0 && in_progress + finish_sync + + in_progress = false + end GraphqlTriggers.security_policies_sync_updated( policy_configuration, @@ -238,7 +295,8 @@ def trigger_subscription projects_total, all_failed_projects, get_progress(merge_requests_pending, merge_requests_total), - merge_requests_total + merge_requests_total, + in_progress ) end @@ -264,8 +322,12 @@ def get_items(key) end end - def with_redis(&block) - Gitlab::Redis::SharedState.with(&block) # rubocop:disable CodeReuse/ActiveRecord -- false positive + def with_redis(conn = nil, &block) + if conn + yield(conn) + else + Gitlab::Redis::SharedState.with(&block) # rubocop:disable CodeReuse/ActiveRecord -- false positive + end end end @@ -275,7 +337,8 @@ def clear_policy_sync_state(config_id) end def append_projects_to_sync(config_id, project_ids) - State.new(config_id).append_projects(project_ids) + state = State.new(config_id) + state.append_projects(project_ids) end def finish_project_policy_sync(project_id) diff --git a/ee/spec/graphql/graphql_triggers_spec.rb b/ee/spec/graphql/graphql_triggers_spec.rb index e6115a6fc92c10b2e639e4c38c9f3b9d8f3e522b..4810d0257f0df500f600d70221f49f5841a1a16c 100644 --- a/ee/spec/graphql/graphql_triggers_spec.rb +++ b/ee/spec/graphql/graphql_triggers_spec.rb @@ -197,7 +197,8 @@ projects_total, failed_projects, merge_requests, - merge_requests_total + merge_requests_total, + in_progress ) end @@ -207,6 +208,7 @@ let(:failed_projects) { [non_existing_record_id.to_s] } let(:merge_requests) { 50.0 } let(:merge_requests_total) { 200 } + let(:in_progress) { true } specify do expect(GitlabSchema.subscriptions).to receive(:trigger).with( @@ -217,7 +219,8 @@ projects_total: projects_total, failed_projects: failed_projects, merge_requests_progress: merge_requests, - merge_requests_total: merge_requests_total + merge_requests_total: merge_requests_total, + in_progress: in_progress } ).and_call_original diff --git a/ee/spec/graphql/subscriptions/security/policies_sync_updated_spec.rb b/ee/spec/graphql/subscriptions/security/policies_sync_updated_spec.rb index 9492f71de95f33d767abc20261763b0990e1378a..df93db35d0695a00419b68ca7f87373b4ecc36f4 100644 --- a/ee/spec/graphql/subscriptions/security/policies_sync_updated_spec.rb +++ b/ee/spec/graphql/subscriptions/security/policies_sync_updated_spec.rb @@ -15,6 +15,7 @@ let(:failed_projects) { ["123"] } let(:merge_requests_progress) { 50 } let(:merge_requests_total) { 200 } + let(:in_progress) { true } let(:subscribe) { security_policies_sync_updated_subscription(policy_configuration, current_user) } @@ -33,7 +34,8 @@ projects_total, failed_projects, merge_requests_progress, - merge_requests_total) + merge_requests_total, + in_progress) end end @@ -48,7 +50,8 @@ "projectsTotal" => projects_total, "failedProjects" => failed_projects, "mergeRequestsProgress" => merge_requests_progress, - "mergeRequestsTotal" => merge_requests_total + "mergeRequestsTotal" => merge_requests_total, + "inProgress" => in_progress }) end end diff --git a/ee/spec/graphql/types/query_type_spec.rb b/ee/spec/graphql/types/query_type_spec.rb index be93573550d5548d39e985ad2a06e1c686517054..515bc6b2c16386412b3a7f92e5ce4379a7ea4982 100644 --- a/ee/spec/graphql/types/query_type_spec.rb +++ b/ee/spec/graphql/types/query_type_spec.rb @@ -65,6 +65,7 @@ :project_secrets_manager, :project_secrets, :secret_permissions, + :security_policies_sync_status, :project_secret, :ai_feature_settings, :ai_slash_commands, diff --git a/ee/spec/lib/security/security_orchestration_policies/policy_sync_state/state_spec.rb b/ee/spec/lib/security/security_orchestration_policies/policy_sync_state/state_spec.rb index ca1f72d3a311f32b2945635b3c6efcc9f438e6d5..daee19cab5938296e941720c45499c97fc305fb6 100644 --- a/ee/spec/lib/security/security_orchestration_policies/policy_sync_state/state_spec.rb +++ b/ee/spec/lib/security/security_orchestration_policies/policy_sync_state/state_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Security::SecurityOrchestrationPolicies::PolicySyncState::State, :clean_gitlab_redis_shared_state, feature_category: :security_policy_management do + let(:redis) { Gitlab::Redis::SharedState.pool.checkout } + let(:merge_request_id) { 1 } let(:project_id) { 1 } let(:other_project_id) { 2 } @@ -34,6 +36,32 @@ end end + describe '#start_sync' do + it 'marks pending' do + expect { state.start_sync }.to change { state.sync_in_progress?(redis) }.from(false).to(true) + end + + context 'with feature disabled' do + before do + stub_feature_flags(security_policy_sync_propagation_tracking: false) + end + + it 'does not mark pending' do + expect { state.start_sync }.not_to change { state.sync_in_progress?(redis) }.from(false) + end + end + end + + describe '#finish_sync' do + before do + state.start_sync + end + + it 'removes pending' do + expect { state.finish_sync }.to change { state.sync_in_progress?(redis) }.from(true).to(false) + end + end + describe '#append_projects' do context 'when adding new project IDs' do it 'adds project IDs as pending' do @@ -352,7 +380,7 @@ end describe '#sync_in_progress?' do - subject(:sync_in_progress?) { state.sync_in_progress? } + subject(:sync_in_progress?) { state.sync_in_progress?(redis) } it { is_expected.to be(false) } @@ -459,17 +487,19 @@ 10, # projects total [], # no failed projects yet 0, # merge request progress: (10 total - 10 pending) / 10 * 100 = 0% - 10 # merge requests total + 10, # merge requests total, + true # in progress ).ordered state.finish_project(1) expect(GraphqlTriggers).to receive(:security_policies_sync_updated).with( policy_configuration, - 10, # project progress: still 10% - 10, # projects total - [], # no failed projects yet - 10, # merge request progress: (10 total - 9 pending) / 10 * 100 = 10% - 10 # merge requests total + 10, # project progress: still 10% + 10, # projects total + [], # no failed projects yet + 10, # merge request progress: (10 total - 9 pending) / 10 * 100 = 10% + 10, # merge requests total, + true # in progress ).ordered state.finish_merge_request_worker(1) @@ -479,7 +509,8 @@ 10, # projects total ["2"], # failed project 2 10, # merge request progress: still 10% - 10 # merge requests total + 10, # merge requests total, + true # in progress ).ordered state.fail_project(2) end @@ -492,11 +523,12 @@ expect(GraphqlTriggers).to have_received(:security_policies_sync_updated).with( policy_configuration, - 50, # project progress: (10 - 5) / 10 * 100 = 50% - 10, # projects total - [], # no failed projects - 0, # merge request progress: still 0% - 10 # merge requests total + 50, # project progress: (10 - 5) / 10 * 100 = 50% + 10, # projects total + [], # no failed projects + 0, # merge request progress: still 0% + 10, # merge requests total, + true # in progress ) end diff --git a/ee/spec/requests/api/graphql/security/policies_sync_status_spec.rb b/ee/spec/requests/api/graphql/security/policies_sync_status_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..af689339f9811b2813bbb977f065f8f27ddbe70a --- /dev/null +++ b/ee/spec/requests/api/graphql/security/policies_sync_status_spec.rb @@ -0,0 +1,136 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe 'querying security policy sync status', feature_category: :security_policy_management do + include GraphqlHelpers + + include_context 'with policy sync state' + + let_it_be(:current_user) { policy_configuration.security_policy_management_project.owner } + + let(:query) do + <<~QUERY + query { + securityPoliciesSyncStatus(policyConfigurationId: "#{policy_configuration.to_global_id}") { + projectsProgress + projectsTotal + failedProjects + mergeRequestsProgress + mergeRequestsTotal + inProgress + } + } + QUERY + end + + before do + policy_configuration.security_policy_management_project.add_maintainer(current_user) + + stub_licensed_features(security_orchestration_policies: true) + end + + subject(:graphql_response) do + graphql_data.fetch("securityPoliciesSyncStatus")&.transform_keys(&:underscore)&.symbolize_keys + end + + context 'without sync' do + specify do + post_graphql(query, current_user: current_user) + + expect(graphql_response).to eq( + projects_progress: 0.0, + projects_total: 0, + failed_projects: [], + merge_requests_progress: 0.0, + merge_requests_total: 0, + in_progress: false + ) + end + end + + context 'with ongoing sync' do + before do + state.start_sync + + state.append_projects([1, 2, 3, 4]) + + state.finish_project(1) + state.fail_project(2) + + state.start_merge_request(1) + state.start_merge_request(2) + + state.start_merge_request_worker(1) + state.finish_merge_request_worker(1) + end + + specify do + post_graphql(query, current_user: current_user) + + expect(graphql_response).to eq( + projects_progress: 50.0, + projects_total: 4, + failed_projects: %w[2], + merge_requests_progress: 50.0, + merge_requests_total: 2, + in_progress: true + ) + end + end + + context 'with completed sync' do + before do + state.start_sync + + state.append_projects([1, 2]) + + state.start_merge_request(1) + state.start_merge_request(2) + + state.start_merge_request_worker(1) + state.finish_merge_request_worker(1) + + state.start_merge_request_worker(2) + state.finish_merge_request_worker(2) + + state.finish_project(1) + state.fail_project(2) + + state.finish_sync + end + + specify do + post_graphql(query, current_user: current_user) + + expect(graphql_response).to eq( + projects_progress: 100.0, + projects_total: 2, + failed_projects: %w[2], + merge_requests_progress: 100.0, + merge_requests_total: 2, + in_progress: false + ) + end + end + + context 'when unauthorized' do + let_it_be(:current_user) { create(:user) } + + before do + post_graphql(query, current_user: current_user) + end + + it { is_expected.to be_nil } + end + + context 'with feature disabled' do + before do + stub_feature_flags(security_policy_sync_propagation_tracking: false) + + post_graphql(query, current_user: current_user) + end + + it { is_expected.to be_nil } + end +end diff --git a/ee/spec/support/helpers/graphql/subscriptions/security/policies_sync_updated/helper.rb b/ee/spec/support/helpers/graphql/subscriptions/security/policies_sync_updated/helper.rb index 87bd6464c6a3ce60fe6678b0923b1dcd58b9a7e1..a36bd3370d518cc508fda36c1320d0d1ae8ab2b5 100644 --- a/ee/spec/support/helpers/graphql/subscriptions/security/policies_sync_updated/helper.rb +++ b/ee/spec/support/helpers/graphql/subscriptions/security/policies_sync_updated/helper.rb @@ -31,6 +31,7 @@ def security_policies_sync_updated_subscription_query(policy_configuration) failedProjects mergeRequestsProgress mergeRequestsTotal + inProgress } } SUBSCRIPTION