diff --git a/config/initializers/click_house.rb b/config/initializers/click_house.rb index 7fc216d9a59469651dd6ce22515df6913f667e6c..c1bec683c6b1f03fd08b8fd71293d4152b6ca883 100644 --- a/config/initializers/click_house.rb +++ b/config/initializers/click_house.rb @@ -3,6 +3,7 @@ return unless File.exist?(Rails.root.join('config/click_house.yml')) raw_config = Rails.application.config_for(:click_house) + return if raw_config.blank? ClickHouse::Client.configure do |config| diff --git a/db/click_house/main/20230705124511_create_events.sql b/db/click_house/main/20230705124511_create_events.sql new file mode 100644 index 0000000000000000000000000000000000000000..45e0139165af53a0262414b26905791ebe6941ea --- /dev/null +++ b/db/click_house/main/20230705124511_create_events.sql @@ -0,0 +1,15 @@ +CREATE TABLE events +( + id UInt64 DEFAULT 0, + path String DEFAULT '', + author_id UInt64 DEFAULT 0, + target_id UInt64 DEFAULT 0, + target_type LowCardinality(String) DEFAULT '', + action UInt8 DEFAULT 0, + created_at DateTime64(6, 'UTC') DEFAULT now(), + updated_at DateTime64(6, 'UTC') DEFAULT now() +) +ENGINE = ReplacingMergeTree(updated_at) +PRIMARY KEY (id) +ORDER BY (id) +PARTITION BY toYear(created_at) diff --git a/gems/click_house-client/lib/click_house/client.rb b/gems/click_house-client/lib/click_house/client.rb index 1c8da64f38f7002616f7fe23b99db9a053a5032f..22c42d7be6eaa33893723d26f0cefd2f03aab655 100644 --- a/gems/click_house-client/lib/click_house/client.rb +++ b/gems/click_house-client/lib/click_house/client.rb @@ -25,9 +25,9 @@ def configure ConfigurationError = Class.new(Error) DatabaseError = Class.new(Error) - def self.execute(query, database, configuration = self.configuration) - db = configuration.databases[database] - raise ConfigurationError, "The database '#{database}' is not configured" unless db + # Executes a SELECT database query + def self.select(query, database, configuration = self.configuration) + db = lookup_database(configuration, database) response = configuration.http_post_proc.call( db.uri.to_s, @@ -39,5 +39,26 @@ def self.execute(query, database, configuration = self.configuration) Formatter.format(configuration.json_parser.parse(response.body)) end + + # Executes any kinds of database query without returning any data (INSERT, DELETE) + def self.execute(query, database, configuration = self.configuration) + db = lookup_database(configuration, database) + + response = configuration.http_post_proc.call( + db.uri.to_s, + db.headers, + query + ) + + raise DatabaseError, response.body unless response.success? + + true + end + + private_class_method def self.lookup_database(configuration, database) + configuration.databases[database].tap do |db| + raise ConfigurationError, "The database '#{database}' is not configured" unless db + end + end end end diff --git a/gems/click_house-client/spec/click_house/client_spec.rb b/gems/click_house-client/spec/click_house/client_spec.rb index 19850bc722745d43f51f4172669d133991dc2f69..883199198badbb1329ebc65ffb5648ef78b68c92 100644 --- a/gems/click_house-client/spec/click_house/client_spec.rb +++ b/gems/click_house-client/spec/click_house/client_spec.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true RSpec.describe ClickHouse::Client do - describe '#execute' do + describe '#select' do # Assuming we have a DB table with the following schema # # CREATE TABLE issues ( @@ -39,7 +39,7 @@ end it 'parses the results and returns the data as array of hashes' do - result = described_class.execute('SELECT * FROM issues', :test_db, configuration) + result = described_class.select('SELECT * FROM issues', :test_db, configuration) timestamp1 = ActiveSupport::TimeZone["UTC"].parse('2023-06-21 13:33:44') timestamp2 = ActiveSupport::TimeZone["UTC"].parse('2023-06-21 13:33:50') @@ -73,7 +73,7 @@ context 'when the DB is not configured' do it 'raises erro' do expect do - described_class.execute('SELECT * FROM issues', :different_db, configuration) + described_class.select('SELECT * FROM issues', :different_db, configuration) end.to raise_error(ClickHouse::Client::ConfigurationError, /not configured/) end end @@ -90,7 +90,7 @@ it 'raises error' do expect do - described_class.execute('SELECT * FROM issues', :test_db, configuration) + described_class.select('SELECT * FROM issues', :test_db, configuration) end.to raise_error(ClickHouse::Client::DatabaseError, 'some error') end end diff --git a/spec/lib/gitlab/database/click_house_client_spec.rb b/spec/lib/gitlab/database/click_house_client_spec.rb index 16b7fb82c4ad985919c2e7efd50f85ee4b64c60e..502d879bf6a32feb4cc123aac346195b9d86d581 100644 --- a/spec/lib/gitlab/database/click_house_client_spec.rb +++ b/spec/lib/gitlab/database/click_house_client_spec.rb @@ -24,10 +24,90 @@ expect(databases).not_to be_empty end - it 'returns data from the DB' do - result = ClickHouse::Client.execute("SELECT 1 AS value", :main) + it 'returns data from the DB via `select` method' do + result = ClickHouse::Client.select("SELECT 1 AS value", :main) + # returns JSON if successful. Otherwise error expect(result).to eq([{ 'value' => 1 }]) end + + it 'does not return data via `execute` method' do + result = ClickHouse::Client.execute("SELECT 1 AS value", :main) + + # does not return data, just true if successful. Otherwise error. + expect(result).to eq(true) + end + + describe 'data manipulation' do + describe 'inserting' do + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project) } + + let_it_be(:author1) { create(:user).tap { |u| project.add_developer(u) } } + let_it_be(:author2) { create(:user).tap { |u| project.add_developer(u) } } + + let_it_be(:issue1) { create(:issue, project: project) } + let_it_be(:issue2) { create(:issue, project: project) } + let_it_be(:merge_request) { create(:merge_request, source_project: project) } + + let_it_be(:event1) { create(:event, :created, target: issue1, author: author1) } + let_it_be(:event2) { create(:event, :closed, target: issue2, author: author2) } + let_it_be(:event3) { create(:event, :merged, target: merge_request, author: author1) } + + let(:events) { [event1, event2, event3] } + + def format_row(event) + path = event.project.reload.project_namespace.traversal_ids.join('/') + + action = Event.actions[event.action] + [ + event.id, + "'#{path}/'", + event.author_id, + event.target_id, + "'#{event.target_type}'", + action, + event.created_at.to_f, + event.updated_at.to_f + ].join(',') + end + + describe 'RSpec hooks' do + it 'ensures that tables are empty' do + results = ClickHouse::Client.select('SELECT * FROM events', :main) + expect(results).to be_empty + end + end + + it 'inserts and modifies data' do + insert_query = <<~SQL + INSERT INTO events + (id, path, author_id, target_id, target_type, action, created_at, updated_at) + VALUES + (#{format_row(event1)}), + (#{format_row(event2)}), + (#{format_row(event3)}) + SQL + + ClickHouse::Client.execute(insert_query, :main) + + results = ClickHouse::Client.select('SELECT * FROM events ORDER BY id', :main) + expect(results.size).to eq(3) + + last = results.last + expect(last).to match(a_hash_including( + 'id' => event3.id, + 'author_id' => event3.author_id, + 'created_at' => be_within(0.05).of(event3.created_at), + 'target_type' => event3.target_type + )) + + ClickHouse::Client.execute("DELETE FROM events WHERE id = #{event3.id}", :main) + + results = ClickHouse::Client.select("SELECT * FROM events WHERE id = #{event3.id}", :main) + expect(results).to be_empty + end + end + end end end diff --git a/spec/support/database/click_house/hooks.rb b/spec/support/database/click_house/hooks.rb new file mode 100644 index 0000000000000000000000000000000000000000..27abd19dc3f23c0b70b089636f1208420f8675e7 --- /dev/null +++ b/spec/support/database/click_house/hooks.rb @@ -0,0 +1,55 @@ +# frozen_string_literal: true + +# rubocop: disable Gitlab/NamespacedClass +class ClickHouseTestRunner + def truncate_tables + ClickHouse::Client.configuration.databases.each_key do |db| + tables_for(db).each do |table| + ClickHouse::Client.execute("TRUNCATE TABLE #{table}", db) + end + end + end + + def ensure_schema + return if @ensure_schema + + ClickHouse::Client.configuration.databases.each_key do |db| + # drop all tables + lookup_tables(db).each do |table| + ClickHouse::Client.execute("DROP TABLE IF EXISTS #{table}", db) + end + + # run the schema SQL files + Dir[Rails.root.join("db/click_house/#{db}/*.sql")].each do |file| + ClickHouse::Client.execute(File.read(file), db) + end + end + + @ensure_schema = true + end + + private + + def tables_for(db) + @tables ||= {} + @tables[db] ||= lookup_tables(db) + end + + def lookup_tables(db) + ClickHouse::Client.select('SHOW TABLES', db).pluck('name') + end +end +# rubocop: enable Gitlab/NamespacedClass + +RSpec.configure do |config| + click_house_test_runner = ClickHouseTestRunner.new + + config.around(:each, :click_house) do |example| + with_net_connect_allowed do + click_house_test_runner.ensure_schema + click_house_test_runner.truncate_tables + + example.run + end + end +end