diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index edb9a2053b12220acb2200cd50534dce60a27f81..2dcee7f64d52b9f430644c80a37e9f038690f82b 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -667,6 +667,7 @@ def self.kroki_formats_attributes attr_encrypted :arkose_labs_public_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) attr_encrypted :arkose_labs_private_api_key, encryption_options_base_32_aes_256_gcm.merge(encode: false, encode_iv: false) attr_encrypted :cube_api_key, encryption_options_base_32_aes_256_gcm + attr_encrypted :jitsu_administrator_password, encryption_options_base_32_aes_256_gcm validates :disable_feed_token, inclusion: { in: [true, false], message: _('must be a boolean value') } diff --git a/config/sidekiq_queues.yml b/config/sidekiq_queues.yml index 49109ec7142724dff746bb06f80a041c0bd87825..0e8f5d4a7d6f0d4f2193fece199074f887fad86a 100644 --- a/config/sidekiq_queues.yml +++ b/config/sidekiq_queues.yml @@ -379,6 +379,8 @@ - 5 - - process_commit - 3 +- - product_analytics_initialize_analytics + - 1 - - project_cache - 1 - - project_destroy diff --git a/db/migrate/20220818125332_add_jitsu_tracking_columns_to_application_settings.rb b/db/migrate/20220818125332_add_jitsu_tracking_columns_to_application_settings.rb new file mode 100644 index 0000000000000000000000000000000000000000..9013168c2c57d9b8a4635b9cbc8bc99dbb186d79 --- /dev/null +++ b/db/migrate/20220818125332_add_jitsu_tracking_columns_to_application_settings.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddJitsuTrackingColumnsToApplicationSettings < Gitlab::Database::Migration[2.0] + def change + # rubocop:disable Migration/AddLimitToTextColumns + # limit is added in 20220818125703_add_jitsu_tracking_columns_to_application_settings_text_limits.rb + add_column :application_settings, :jitsu_host, :text + add_column :application_settings, :jitsu_project_xid, :text + add_column :application_settings, :clickhouse_connection_string, :text + add_column :application_settings, :jitsu_administrator_email, :text + add_column :application_settings, :encrypted_jitsu_administrator_password, :binary + add_column :application_settings, :encrypted_jitsu_administrator_password_iv, :binary + # rubocop:enable Migration/AddLimitToTextColumns + end +end diff --git a/db/migrate/20220818125703_add_jitsu_tracking_columns_to_application_settings_text_limits.rb b/db/migrate/20220818125703_add_jitsu_tracking_columns_to_application_settings_text_limits.rb new file mode 100644 index 0000000000000000000000000000000000000000..41de6e3472412024b560bccca0ab9354eab517c2 --- /dev/null +++ b/db/migrate/20220818125703_add_jitsu_tracking_columns_to_application_settings_text_limits.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddJitsuTrackingColumnsToApplicationSettingsTextLimits < Gitlab::Database::Migration[2.0] + disable_ddl_transaction! + + def up + add_text_limit :application_settings, :jitsu_host, 255 + add_text_limit :application_settings, :jitsu_project_xid, 255 + add_text_limit :application_settings, :clickhouse_connection_string, 1024 + add_text_limit :application_settings, :jitsu_administrator_email, 255 + end + + def down + remove_text_limit :application_settings, :jitsu_host + remove_text_limit :application_settings, :jitsu_project_xid + remove_text_limit :application_settings, :clickhouse_connection_string + remove_text_limit :application_settings, :jitsu_administrator_email + end +end diff --git a/db/schema_migrations/20220818125332 b/db/schema_migrations/20220818125332 new file mode 100644 index 0000000000000000000000000000000000000000..35c76c4318fd04d97dda81ac10775bd0682f60f8 --- /dev/null +++ b/db/schema_migrations/20220818125332 @@ -0,0 +1 @@ +ebcf446aa6579d93c57c2e96e8b670a43bcb6e20216f33a7f535e1bed50ace62 \ No newline at end of file diff --git a/db/schema_migrations/20220818125703 b/db/schema_migrations/20220818125703 new file mode 100644 index 0000000000000000000000000000000000000000..1bfebfc50ad7b200995a89bf6f69ddb3c768056e --- /dev/null +++ b/db/schema_migrations/20220818125703 @@ -0,0 +1 @@ +b60f36cd83174ce257baba4a74f0fcba6cd462fa2af6530ff5a3341536058e12 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index ed7bf933a969aacacd480eb67a398fdbbc37bd8d..3eefc98897a5ed5b4aae173f70ad1cce72580c91 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -11473,6 +11473,12 @@ CREATE TABLE application_settings ( cube_api_base_url text, encrypted_cube_api_key bytea, encrypted_cube_api_key_iv bytea, + jitsu_host text, + jitsu_project_xid text, + clickhouse_connection_string text, + jitsu_administrator_email text, + encrypted_jitsu_administrator_password bytea, + encrypted_jitsu_administrator_password_iv bytea, dashboard_limit_enabled boolean DEFAULT false NOT NULL, dashboard_limit integer DEFAULT 0 NOT NULL, dashboard_notification_limit integer DEFAULT 0 NOT NULL, @@ -11514,12 +11520,16 @@ CREATE TABLE application_settings ( CONSTRAINT check_9c6c447a13 CHECK ((char_length(maintenance_mode_message) <= 255)), CONSTRAINT check_a5704163cc CHECK ((char_length(secret_detection_revocation_token_types_url) <= 255)), CONSTRAINT check_d03919528d CHECK ((char_length(container_registry_vendor) <= 255)), + CONSTRAINT check_d4865d70f3 CHECK ((char_length(clickhouse_connection_string) <= 1024)), CONSTRAINT check_d820146492 CHECK ((char_length(spam_check_endpoint_url) <= 255)), + CONSTRAINT check_dea8792229 CHECK ((char_length(jitsu_host) <= 255)), CONSTRAINT check_e2dd6e290a CHECK ((char_length(jira_connect_application_key) <= 255)), CONSTRAINT check_e5024c8801 CHECK ((char_length(elasticsearch_username) <= 255)), CONSTRAINT check_e5aba18f02 CHECK ((char_length(container_registry_version) <= 255)), + CONSTRAINT check_ec3ca9aa8d CHECK ((char_length(jitsu_administrator_email) <= 255)), CONSTRAINT check_ef6176834f CHECK ((char_length(encrypted_cloud_license_auth_token_iv) <= 255)), - CONSTRAINT check_f6563bc000 CHECK ((char_length(arkose_labs_verify_api_url) <= 255)) + CONSTRAINT check_f6563bc000 CHECK ((char_length(arkose_labs_verify_api_url) <= 255)), + CONSTRAINT check_fc732c181e CHECK ((char_length(jitsu_project_xid) <= 255)) ); COMMENT ON COLUMN application_settings.content_validation_endpoint_url IS 'JiHu-specific column'; diff --git a/ee/app/models/ee/application_setting.rb b/ee/app/models/ee/application_setting.rb index 124c649dd8ad6311ee5dc0f824a61bf6b1008916..f042ad34c28d24c82047a88d1c18ef5b0ef02ad7 100644 --- a/ee/app/models/ee/application_setting.rb +++ b/ee/app/models/ee/application_setting.rb @@ -213,7 +213,12 @@ def defaults max_number_of_repository_downloads: 0, max_number_of_repository_downloads_within_time_period: 0, git_rate_limit_users_allowlist: [], - auto_ban_user_on_excessive_projects_download: false + auto_ban_user_on_excessive_projects_download: false, + jitsu_host: nil, + jitsu_project_xid: nil, + clickhouse_connection_string: nil, + jitsu_administrator_email: nil, + jitsu_administrator_password: nil ) end end diff --git a/ee/app/models/product_analytics/jitsu_authentication.rb b/ee/app/models/product_analytics/jitsu_authentication.rb new file mode 100644 index 0000000000000000000000000000000000000000..bdf6b71b9ea87e4de79c433694e3a1b848a82825 --- /dev/null +++ b/ee/app/models/product_analytics/jitsu_authentication.rb @@ -0,0 +1,95 @@ +# frozen_string_literal: true + +module ProductAnalytics + class JitsuAuthentication + def initialize(jid, project) + @jid = jid + @project = project + + @root_url = "#{Gitlab::CurrentSettings.jitsu_host}/configurator" + @clickhouse_connection_string = Gitlab::CurrentSettings.clickhouse_connection_string + @jitsu_project_xid = Gitlab::CurrentSettings.jitsu_project_xid + @jitsu_administrator_email = Gitlab::CurrentSettings.jitsu_administrator_email + @jitsu_administrator_password = Gitlab::CurrentSettings.jitsu_administrator_password + end + + def create_api_key! + response = Gitlab::HTTP.post( + "#{@root_url}/api/v2/objects/#{@jitsu_project_xid}/api_keys", + allow_local_requests: true, + headers: { + Authorization: "Bearer #{generate_access_token}" + }, + body: { + 'comment': @project.to_global_id.to_s, + 'jsAuth': SecureRandom.uuid + }.to_json + ) + + json = Gitlab::Json.parse(response.body) + + response.success? ? { jsAuth: json['jsAuth'], uid: json['uid'] } : log_jitsu_api_error(json) + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e) + end + + def create_clickhouse_destination! + id = SecureRandom.uuid + + response = Gitlab::HTTP.post( + "#{@root_url}/api/v2/objects/#{@jitsu_project_xid}/destinations", + allow_local_requests: true, + headers: { + Authorization: "Bearer #{generate_access_token}" + }, + body: { + _type: 'clickhouse', + _onlyKeys: [create_api_key![:uid]], + _id: id, + _uid: SecureRandom.uuid, + _connectionTestOk: true, + _formData: { + ch_database: "gitlab_project_#{@project.id}", + mode: 'stream', + tableName: "jitsu", + ch_dsns_list: [@clickhouse_connection_string] + } + }.to_json + ) + + response.success? ? id : log_jitsu_api_error(Gitlab::Json.parse(response.body)) + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e) + end + + def generate_access_token + response = Gitlab::HTTP.post( + "#{@root_url}/api/v1/users/signin", + allow_local_requests: true, + headers: { 'Content-Type' => 'application/json' }, + body: { + 'email': @jitsu_administrator_email, + 'password': @jitsu_administrator_password + }.to_json + ) + + json = Gitlab::Json.parse(response.body) + + response.success? ? json['access_token'] : log_jitsu_api_error(json) + rescue StandardError => e + Gitlab::ErrorTracking.track_exception(e) + end + + private + + def log_jitsu_api_error(json) + Gitlab::AppLogger.error( + message: 'Jitsu API error', + error: json['error'], + jitsu_error_message: json['message'], + project_id: @project.id, + job_id: @jid + ) + end + end +end diff --git a/ee/app/services/product_analytics/initialize_stack_service.rb b/ee/app/services/product_analytics/initialize_stack_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..bfc042b74daeb1b083898ec611765905b1664b2b --- /dev/null +++ b/ee/app/services/product_analytics/initialize_stack_service.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module ProductAnalytics + class InitializeStackService < BaseContainerService + def execute + return unless ::Feature.enabled?(:jitsu_connection_proof_of_concept, container.group) + + ::ProductAnalytics::InitializeAnalyticsWorker.perform_async(container.id) + end + end +end diff --git a/ee/app/workers/all_queues.yml b/ee/app/workers/all_queues.yml index 9874b6ab56b763695bdb30f4457c0449d1ea56a2..ccb8974f0e4371f841fc125412d01fd8186f7b68 100644 --- a/ee/app/workers/all_queues.yml +++ b/ee/app/workers/all_queues.yml @@ -1335,6 +1335,15 @@ :weight: 1 :idempotent: true :tags: [] +- :name: product_analytics_initialize_analytics + :worker_name: ProductAnalytics::InitializeAnalyticsWorker + :feature_category: :product_analytics + :has_external_dependencies: true + :urgency: :low + :resource_boundary: :unknown + :weight: 1 + :idempotent: true + :tags: [] - :name: project_import_schedule :worker_name: ProjectImportScheduleWorker :feature_category: :source_code_management diff --git a/ee/app/workers/product_analytics/initialize_analytics_worker.rb b/ee/app/workers/product_analytics/initialize_analytics_worker.rb new file mode 100644 index 0000000000000000000000000000000000000000..c89aa37e822341c5439d927e2fd6e872348400cf --- /dev/null +++ b/ee/app/workers/product_analytics/initialize_analytics_worker.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module ProductAnalytics + class InitializeAnalyticsWorker + include ApplicationWorker + + data_consistency :sticky + feature_category :product_analytics + idempotent! + worker_has_external_dependencies! + + def perform(project_id) + return if Gitlab::CurrentSettings.jitsu_host.nil? || Gitlab::CurrentSettings.jitsu_project_xid.nil? + + @project = Project.find(project_id) + ProductAnalytics::JitsuAuthentication.new(jid, @project).create_clickhouse_destination! + end + end +end diff --git a/ee/config/feature_flags/development/jitsu_connection_proof_of_concept.yml b/ee/config/feature_flags/development/jitsu_connection_proof_of_concept.yml new file mode 100644 index 0000000000000000000000000000000000000000..7494e073844accf89df4c909f6aaebc146a5f66a --- /dev/null +++ b/ee/config/feature_flags/development/jitsu_connection_proof_of_concept.yml @@ -0,0 +1,8 @@ +--- +name: jitsu_connection_proof_of_concept +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/95999 +rollout_issue_url: +milestone: '15.5' +type: development +group: group::product analytics +default_enabled: false diff --git a/ee/spec/models/product_analytics/jitsu_authentication_spec.rb b/ee/spec/models/product_analytics/jitsu_authentication_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..951a47ca0b2c5b73a43ee7d7458de390f6d4e9ac --- /dev/null +++ b/ee/spec/models/product_analytics/jitsu_authentication_spec.rb @@ -0,0 +1,207 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ProductAnalytics::JitsuAuthentication do + let(:jid) { '12345678' } + let(:error_message) { '' } + let(:jitsu_error_message) { '' } + let_it_be(:project) { create(:project) } + + subject(:auth) { described_class.new(jid, project) } + + before do + stub_application_setting( + jitsu_host: 'http://jitsu.dev', + jitsu_project_xid: 'testtesttesttestprj', + jitsu_administrator_email: 'test@test.com', + jitsu_administrator_password: 'testtest' + ) + end + + shared_examples 'returns nil and logs the API error' do + it do + expect(Gitlab::AppLogger).to receive(:error).with( + message: 'Jitsu API error', + error: error_message, + jitsu_error_message: jitsu_error_message, + project_id: project.id, + job_id: jid + ) + expect(subject).to be_nil + end + end + + shared_examples 'returns nil and logs the exception' do + it do + expect(Gitlab::ErrorTracking).to receive(:track_exception).with(instance_of(Gitlab::HTTP::Error)) + expect(subject).to be_nil + end + end + + describe '#generate_access_token' do + subject { auth.generate_access_token } + + context 'when request is successful' do + before do + stub_signin_success + end + + it { is_expected.to eq('thisisanaccesstoken') } + end + + context 'when request is unsuccessful' do + let(:error_message) { 'invalid password' } + let(:jitsu_error_message) { 'Authorization failed: invalid password' } + + before do + stub_signin_failure + end + + it_behaves_like 'returns nil and logs the API error' + end + + context 'when request throws an exception' do + before do + stub_signin_exception + end + + it_behaves_like 'returns nil and logs the exception' + end + end + + describe '#create_api_key!' do + subject { auth.create_api_key! } + + context 'when request is successful' do + before do + stub_signin_success + stub_api_key_success + allow(auth).to receive(:access_token).and_return('testtoken') + end + + it { is_expected.to eq({ jsAuth: 'Mp1N4PYvRXNk1KIh2MLDE7BYghnSwdnt', uid: 'yijlmncqjot0xy9h6rv54p.s7zz20' }) } + end + + context 'when request is unsuccessful' do + let(:error_message) { 'token required' } + let(:jitsu_error_message) { 'Authorization failed: token required' } + + before do + stub_signin_success + stub_api_key_failure + allow(auth).to receive(:access_token).and_return('testtoken') + end + + it_behaves_like 'returns nil and logs the API error' + end + + context 'when request throws an exception' do + before do + stub_signin_success + stub_api_key_exception + allow(auth).to receive(:access_token).and_return('testtoken') + end + + it_behaves_like 'returns nil and logs the exception' + end + end + + describe '#create_clickhouse_destination' do + subject { auth.create_clickhouse_destination! } + + context 'when request is successful' do + before do + stub_signin_success + stub_api_key_success + stub_clickhouse_success + allow(auth).to receive(:access_token).and_return('testtoken') + end + + it { is_expected.not_to be_nil } + end + + context 'when request is unsuccessful' do + let(:error_message) { 'token required' } + let(:jitsu_error_message) { 'Authorization failed: token required' } + + before do + stub_signin_success + stub_api_key_success + stub_clickhouse_failure + allow(auth).to receive(:access_token).and_return('testtoken') + end + + it_behaves_like 'returns nil and logs the API error' + end + + context 'when request throws an exception' do + before do + stub_signin_success + stub_api_key_success + stub_clickhouse_exception + allow(auth).to receive(:access_token).and_return('testtoken') + end + + it_behaves_like 'returns nil and logs the exception' + end + end + + private + + def stub_signin_success + stub_request(:post, "http://jitsu.dev/configurator/api/v1/users/signin") + .with(body: "{\"email\":\"test@test.com\",\"password\":\"testtest\"}") + .to_return(status: 200, body: { access_token: 'thisisanaccesstoken' }.to_json, headers: {}) + end + + def stub_signin_failure + stub_request(:post, "http://jitsu.dev/configurator/api/v1/users/signin") + .with(body: "{\"email\":\"test@test.com\",\"password\":\"testtest\"}") + .to_return(status: 401, + body: { error: 'invalid password', message: 'Authorization failed: invalid password' }.to_json, + headers: {}) + end + + def stub_signin_exception + stub_request(:post, "http://jitsu.dev/configurator/api/v1/users/signin") + .with(body: "{\"email\":\"test@test.com\",\"password\":\"testtest\"}") + .to_raise(Gitlab::HTTP::Error) + end + + def stub_api_key_success + stub_request(:post, "http://jitsu.dev/configurator/api/v2/objects/testtesttesttestprj/api_keys") + .to_return(status: 200, + body: "{\"jsAuth\":\"Mp1N4PYvRXNk1KIh2MLDE7BYghnSwdnt\",\"uid\":\"yijlmncqjot0xy9h6rv54p.s7zz20\"}", + headers: {}) + end + + def stub_api_key_failure + stub_request(:post, "http://jitsu.dev/configurator/api/v2/objects/testtesttesttestprj/api_keys") + .to_return(status: 401, + body: { error: 'token required', message: 'Authorization failed: token required' }.to_json, + headers: {}) + end + + def stub_api_key_exception + stub_request(:post, "http://jitsu.dev/configurator/api/v2/objects/testtesttesttestprj/api_keys") + .to_raise(Gitlab::HTTP::Error) + end + + def stub_clickhouse_success + stub_request(:post, "http://jitsu.dev/configurator/api/v2/objects/testtesttesttestprj/destinations") + .to_return(status: 200) + end + + def stub_clickhouse_failure + stub_request(:post, "http://jitsu.dev/configurator/api/v2/objects/testtesttesttestprj/destinations") + .to_return(status: 401, + body: { error: 'token required', message: 'Authorization failed: token required' }.to_json, + headers: {}) + end + + def stub_clickhouse_exception + stub_request(:post, "http://jitsu.dev/configurator/api/v2/objects/testtesttesttestprj/destinations") + .to_raise(Gitlab::HTTP::Error) + end +end diff --git a/ee/spec/services/product_analytics/initialize_stack_service_spec.rb b/ee/spec/services/product_analytics/initialize_stack_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..09cccf54486e30848992eea4ce89465e2c255225 --- /dev/null +++ b/ee/spec/services/product_analytics/initialize_stack_service_spec.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ProductAnalytics::InitializeStackService do + let_it_be(:project) { create(:project) } + + describe '#execute' do + subject { described_class.new(container: project).execute } + + context 'when feature flag is enabled' do + it 'enqueues a job' do + expect(::ProductAnalytics::InitializeAnalyticsWorker).to receive(:perform_async).with(project.id) + + subject + end + end + + context 'when feature flag is disabled' do + before do + stub_feature_flags(jitsu_connection_proof_of_concept: false) + end + + it 'does not enqueue a job' do + expect(::ProductAnalytics::InitializeAnalyticsWorker).not_to receive(:perform_async) + + subject + end + end + end +end diff --git a/ee/spec/workers/product_analytics/initialize_analytics_worker_spec.rb b/ee/spec/workers/product_analytics/initialize_analytics_worker_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..165e9001f3fea2e54038a2e3127594c44eade4eb --- /dev/null +++ b/ee/spec/workers/product_analytics/initialize_analytics_worker_spec.rb @@ -0,0 +1,69 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe ProductAnalytics::InitializeAnalyticsWorker do + let(:jid) { '12345678' } + let_it_be(:project) { create(:project) } + + subject(:worker) { described_class.new } + + before do + allow(worker).to receive(:jid).and_return(jid) + end + + shared_examples 'a worker that did not make any HTTP calls' do + it 'makes no HTTP calls to the Jitsu configurator API' do + subject + + expect(Gitlab::HTTP).not_to receive(:post) + end + end + + describe 'perform' do + subject { worker.perform(project.id) } + + context 'when jitsu_host application setting is not defined' do + before do + stub_application_setting(jitsu_host: nil) + end + + it_behaves_like 'a worker that did not make any HTTP calls' + end + + context 'when jitsu_project_xid application setting is not defined' do + before do + stub_application_setting(jitsu_project_xid: nil) + end + + it_behaves_like 'a worker that did not make any HTTP calls' + end + + context 'when all application settings are defined' do + before do + stub_application_setting( + jitsu_host: 'http://jitsu.dev', + jitsu_project_xid: 'testtesttesttestprj', + jitsu_administrator_email: 'test@test.com', + jitsu_administrator_password: 'testtest' + ) + end + + it 'sends a HTTP request to create a clickhouse destination' do + expect_next_instance_of(ProductAnalytics::JitsuAuthentication) do |auth| + expect(auth).to receive(:create_clickhouse_destination!).once + end + + subject + end + + context 'when project does not exist' do + subject { worker.perform(non_existing_record_id) } + + it 'raises a RecordNotFound error' do + expect { subject }.to raise_error(ActiveRecord::RecordNotFound) + end + end + end + end +end