diff --git a/doc/administration/audit_event_streaming.md b/doc/administration/audit_event_streaming.md index 3bdc67e5a69cd69bd417ace72207f99b20f4d6fa..4f2fc8c26e25e2f1e0dc38188650c6a4f136e92c 100644 --- a/doc/administration/audit_event_streaming.md +++ b/doc/administration/audit_event_streaming.md @@ -78,3 +78,184 @@ token is generated when the event destination is created and cannot be changed. Each streamed event contains a random alphanumeric identifier for the `X-Gitlab-Event-Streaming-Token` HTTP header that can be verified against the destination's value when [listing streaming destinations](#list-currently-enabled-streaming-destinations). + +## Audit event streaming on Git operations + +> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/332747) in GitLab 14.9 [with a flag](../administration/feature_flags.md) named `audit_event_streaming_git_operations`. Disabled by default. + +FLAG: +On self-managed GitLab, by default this feature is not available. To make it available, ask an administrator to [enable the feature flag](feature_flags.md) named `audit_event_streaming_git_operations`. On GitLab.com, this feature is not available. + +Streaming audit events can be sent when signed-in users push or pull a project's remote Git repositories: + +- [Using SSH](../ssh/index.md). +- Using HTTP or HTTPS. +- Using the **Download** button (**{download}**) in GitLab UI. + +Audit events are not captured for users that are not signed in. For example, when downloading a public project. + +To configure streaming audit events for Git operations, see [Add a new event streaming destination](#add-a-new-event-streaming-destination). + +### Request headers + +Request headers are formatted as follows: + +```plaintext +POST /logs HTTP/1.1 +Host: +Content-Type: application/x-www-form-urlencoded +X-Gitlab-Event-Streaming-Token: +``` + +### Example responses for SSH events + +Fetch: + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "Administrator", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "ssh", + "action": "git-upload-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "Administrator", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-02-23T06:21:05.283Z", + "target_type": "Project", + "target_id": 29 +} +``` + +Push: + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "Administrator", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "ssh", + "action": "git-receive-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "Administrator", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-02-23T06:23:08.746Z", + "target_type": "Project", + "target_id": 29 +} +``` + +### Example responses for HTTP and HTTPS events + +Fetch: + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "Administrator", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "http", + "action": "git-upload-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "Administrator", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-02-23T06:25:43.938Z", + "target_type": "Project", + "target_id": 29 +} +``` + +Push: + +```json +{ + "id": 1, + "author_id": 1, + "entity_id": 29, + "entity_type": "Project", + "details": { + "author_name": "Administrator", + "target_id": 29, + "target_type": "Project", + "target_details": "example-project", + "custom_message": { + "protocol": "http", + "action": "git-receive-pack" + }, + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "Administrator", + "entity_path": "example-group/example-project", + "target_details": "example-project", + "created_at": "2022-02-23T06:26:29.294Z", + "target_type": "Project", + "target_id": 29 +} +``` + +### Example responses for events from GitLab UI download button + +Fetch: + +```json +{ + "id": 1, + "author_id": 99, + "entity_id": 29, + "entity_type": "Project", + "details": { + "custom_message": "Repository Download Started", + "author_name": "example_username", + "target_id": 29, + "target_type": "Project", + "target_details": "example-group/example-project", + "ip_address": "127.0.0.1", + "entity_path": "example-group/example-project" + }, + "ip_address": "127.0.0.1", + "author_name": "example_username", + "entity_path": "example-group/example-project", + "target_details": "example-group/example-project", + "created_at": "2022-02-23T06:27:17.873Z", + "target_type": "Project", + "target_id": 29 +} +``` diff --git a/ee/app/controllers/ee/repositories/git_http_controller.rb b/ee/app/controllers/ee/repositories/git_http_controller.rb index 903fec8f0c74c9aa4c751e79526cf3d0d595ca2b..e7b940324183b43602fcba6e36616bcf2b5530f9 100644 --- a/ee/app/controllers/ee/repositories/git_http_controller.rb +++ b/ee/app/controllers/ee/repositories/git_http_controller.rb @@ -81,6 +81,12 @@ def authenticate_user render_bad_geo_jwt("Invalid signature time ") end + override :update_fetch_statistics + def update_fetch_statistics + send_git_audit_streaming_event + super + end + def jwt_scope_valid? decoded_authorization[:scope] == repository_path.delete_suffix('.git') end @@ -102,6 +108,18 @@ def render_bad_geo_response(message) def ip_allowed? ::Gitlab::Geo.allowed_ip?(request.ip) end + + def send_git_audit_streaming_event + return if user.blank? || project.blank? + return unless ::Feature.enabled?(:audit_event_streaming_git_operations, project.group) + + AuditEvents::BuildService.new( + author: user, + scope: project, + target: project, + message: { protocol: 'http', action: 'git-upload-pack' } + ).execute.stream_to_external_destinations(use_json: true) + end end end end diff --git a/ee/app/models/ee/audit_event.rb b/ee/app/models/ee/audit_event.rb index 9aa269f9a4884e726a87f12cbdad1caa02fc9b42..b3d76cc65358224482ee99adebe89ca841af8549 100644 --- a/ee/app/models/ee/audit_event.rb +++ b/ee/app/models/ee/audit_event.rb @@ -54,11 +54,12 @@ def lazy_entity end end - def stream_to_external_destinations + def stream_to_external_destinations(use_json: false) return if entity.nil? return unless group_entity&.licensed_feature_available?(:external_audit_events) - AuditEvents::AuditEventStreamingWorker.perform_async(id) + perform_params = use_json ? [nil, self.to_json] : [id, nil] + AuditEvents::AuditEventStreamingWorker.perform_async(*perform_params) end def entity_is_group_or_project? diff --git a/ee/app/workers/audit_events/audit_event_streaming_worker.rb b/ee/app/workers/audit_events/audit_event_streaming_worker.rb index 6eaee43bd02af537adcf1484e45bd20ca85cd778..151b4530abb5215d0c7137c24dc6505a8bf00305 100644 --- a/ee/app/workers/audit_events/audit_event_streaming_worker.rb +++ b/ee/app/workers/audit_events/audit_event_streaming_worker.rb @@ -46,7 +46,7 @@ def audit_event(audit_event_id, audit_event_json) end def parse_audit_event_json(audit_event_json) - audit_event_json = Gitlab::Json.parse(audit_event_json) + audit_event_json = Gitlab::Json.parse(audit_event_json).with_indifferent_access audit_event = AuditEvent.new(audit_event_json) # We want to have created_at as unique id for deduplication if audit_event id is not present audit_event.id = audit_event.created_at.to_i if audit_event.id.blank? diff --git a/ee/config/feature_flags/development/audit_event_streaming_git_operations.yml b/ee/config/feature_flags/development/audit_event_streaming_git_operations.yml new file mode 100644 index 0000000000000000000000000000000000000000..c12dad37bf44df1947e8ba2f4f92a0f91b4c666a --- /dev/null +++ b/ee/config/feature_flags/development/audit_event_streaming_git_operations.yml @@ -0,0 +1,8 @@ +--- +name: audit_event_streaming_git_operations +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/76719 +rollout_issue_url: +milestone: '14.9' +type: development +group: group::compliance +default_enabled: false diff --git a/ee/lib/ee/api/internal/base.rb b/ee/lib/ee/api/internal/base.rb index d70c6edb3a8ad33028d2a881e2257133a9f041d5..4cebd645c77d25902caea83ec8b14b422f41fe23 100644 --- a/ee/lib/ee/api/internal/base.rb +++ b/ee/lib/ee/api/internal/base.rb @@ -23,6 +23,19 @@ def check_allowed(params) end end + override :send_git_audit_streaming_event + def send_git_audit_streaming_event(msg) + return if actor.user.blank? || @project.blank? + return unless ::Feature.enabled?(:audit_event_streaming_git_operations, @project.group) + + ::AuditEvents::BuildService.new( + author: actor.user, + scope: @project, + target: @project, + message: msg + ).execute.stream_to_external_destinations(use_json: true) + end + override :two_factor_otp_check def two_factor_otp_check return { success: false, message: 'Feature is not available' } unless ::License.feature_available?(:git_two_factor_enforcement) diff --git a/ee/spec/controllers/projects/repositories_controller_spec.rb b/ee/spec/controllers/projects/repositories_controller_spec.rb index 8d9290c1063c60a4aed3153a15acd2a6133d1cbe..7325e8d699b5e77cc362397ea637340528e43298 100644 --- a/ee/spec/controllers/projects/repositories_controller_spec.rb +++ b/ee/spec/controllers/projects/repositories_controller_spec.rb @@ -3,14 +3,22 @@ require "spec_helper" RSpec.describe Projects::RepositoriesController do - let(:project) { create(:project, :repository) } + let(:group) { create(:group) } + let(:project) { create(:project, :repository, namespace: group) } describe "GET archive" do + subject(:get_archive) do + get :archive, params: { namespace_id: project.namespace, project_id: project, id: "master" }, format: "zip" + end + + def set_group_destination + group.external_audit_event_destinations.create!(destination_url: 'http://example.com') + stub_licensed_features(external_audit_events: true) + end + shared_examples 'logs the audit event' do it 'logs the audit event' do - expect do - get :archive, params: { namespace_id: project.namespace, project_id: project, id: "master" }, format: "zip" - end.to change { AuditEvent.count }.by(1) + expect { get_archive }.to change { AuditEvent.count }.by(1) end end @@ -18,6 +26,16 @@ it_behaves_like 'logs the audit event' do let(:project) { create(:project, :repository, :public) } end + + context 'when group sets event destination' do + before do + set_group_destination + end + it "doesn't send the streaming audit event" do + expect(AuditEvents::AuditEventStreamingWorker).not_to receive(:perform_async) + get_archive + end + end end context 'when authenticated', 'as a developer' do @@ -29,6 +47,18 @@ it_behaves_like 'logs the audit event' do let(:user) { create(:user) } end + + context 'when group sets event destination' do + let(:user) { create(:user) } + + before do + set_group_destination + end + it "sends the streaming audit event" do + expect(AuditEvents::AuditEventStreamingWorker).to receive(:perform_async) + get_archive + end + end end end end diff --git a/ee/spec/controllers/repositories/git_http_controller_spec.rb b/ee/spec/controllers/repositories/git_http_controller_spec.rb index 2f01da29b719dbf7fbce011d34fa2ffc0c7b5804..8379b2b2307b34e34585b6291b8d498078d1cf7d 100644 --- a/ee/spec/controllers/repositories/git_http_controller_spec.rb +++ b/ee/spec/controllers/repositories/git_http_controller_spec.rb @@ -22,4 +22,14 @@ let(:access_checker_class) { Gitlab::GitAccessWiki } end end + + context 'git audit streaming event' do + include GitHttpHelpers + + it_behaves_like 'sends git audit streaming event' do + subject do + post :git_upload_pack, params: { repository_path: "#{project.full_path}.git" } + end + end + end end diff --git a/ee/spec/requests/api/internal/base_spec.rb b/ee/spec/requests/api/internal/base_spec.rb index 48a47715952cf4030fe401b3374028dc9fd9b365..c73d149d34f344ab40ec84206ca38899481fa038 100644 --- a/ee/spec/requests/api/internal/base_spec.rb +++ b/ee/spec/requests/api/internal/base_spec.rb @@ -277,6 +277,12 @@ def check_access_by_alias(alias_name) end end end + + context 'git audit streaming event' do + it_behaves_like 'sends git audit streaming event' do + subject { pull(key, project) } + end + end end describe "POST /internal/lfs_authenticate", :geo do diff --git a/lib/api/internal/base.rb b/lib/api/internal/base.rb index 48157a91477d2c83ddc1f22f68666aa5abac6ff2..9c527f28d445df2e1eaee629b5f6b408205a60c3 100644 --- a/lib/api/internal/base.rb +++ b/lib/api/internal/base.rb @@ -92,6 +92,8 @@ def check_allowed(params) payload[:git_config_options] << "receive.maxInputSize=#{receive_max_input_size.megabytes}" end + send_git_audit_streaming_event(protocol: params[:protocol], action: params[:action]) + response_with_status(**payload) when ::Gitlab::GitAccessResult::CustomAction response_with_status(code: 300, payload: check_result.payload, gl_console_messages: check_result.console_messages) @@ -100,6 +102,10 @@ def check_allowed(params) end end + def send_git_audit_streaming_event(msg) + # Defined in EE + end + def access_check!(actor, params) access_checker = access_checker_for(actor, params[:protocol]) access_checker.check(params[:action], params[:changes]).tap do |result| diff --git a/spec/support/shared_examples/sends_git_audit_streaming_event_shared_examples.rb b/spec/support/shared_examples/sends_git_audit_streaming_event_shared_examples.rb new file mode 100644 index 0000000000000000000000000000000000000000..2c2be0152a07b515910573ab28498d33cdfb1d6b --- /dev/null +++ b/spec/support/shared_examples/sends_git_audit_streaming_event_shared_examples.rb @@ -0,0 +1,61 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'sends git audit streaming event' do + let_it_be(:user) { create(:user) } + + before do + stub_licensed_features(external_audit_events: true) + end + + subject {} + + context 'for public groups and projects' do + let(:group) { create(:group, :public) } + let(:project) { create(:project, :public, :repository, namespace: group) } + + before do + group.external_audit_event_destinations.create!(destination_url: 'http://example.com') + project.add_developer(user) + end + + context 'when user not logged in' do + let(:key) { create(:key) } + + before do + if request + request.headers.merge! auth_env(user.username, nil, nil) + end + end + it 'sends the audit streaming event' do + expect(AuditEvents::AuditEventStreamingWorker).not_to receive(:perform_async) + subject + end + end + end + + context 'for private groups and projects' do + let(:group) { create(:group, :private) } + let(:project) { create(:project, :private, :repository, namespace: group) } + + before do + group.external_audit_event_destinations.create!(destination_url: 'http://example.com') + project.add_developer(user) + sign_in(user) + end + + context 'when user logged in' do + let(:key) { create(:key, user: user) } + + before do + if request + password = user.try(:password) || user.try(:token) + request.headers.merge! auth_env(user.username, password, nil) + end + end + it 'sends the audit streaming event' do + expect(AuditEvents::AuditEventStreamingWorker).to receive(:perform_async).once + subject + end + end + end +end