From af5839be07d635908e27bc4b4df58a9581688448 Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 10 Mar 2022 14:44:47 +0100 Subject: [PATCH 1/4] Move the UsageCounter::PodLogs spec to FOSS This feature was moved to Core in https://gitlab.com/gitlab-org/gitlab/-/merge_requests/26383 but we forgot this spec. --- {ee/spec => spec}/lib/gitlab/usage_counters/pod_logs_spec.rb | 0 .../gitlab/usage_data_counters}/usage_counter_shared_examples.rb | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename {ee/spec => spec}/lib/gitlab/usage_counters/pod_logs_spec.rb (100%) rename {ee/spec/support/shared_examples/lib/gitlab/usage_counters => spec/support/shared_examples/lib/gitlab/usage_data_counters}/usage_counter_shared_examples.rb (100%) diff --git a/ee/spec/lib/gitlab/usage_counters/pod_logs_spec.rb b/spec/lib/gitlab/usage_counters/pod_logs_spec.rb similarity index 100% rename from ee/spec/lib/gitlab/usage_counters/pod_logs_spec.rb rename to spec/lib/gitlab/usage_counters/pod_logs_spec.rb diff --git a/ee/spec/support/shared_examples/lib/gitlab/usage_counters/usage_counter_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/usage_data_counters/usage_counter_shared_examples.rb similarity index 100% rename from ee/spec/support/shared_examples/lib/gitlab/usage_counters/usage_counter_shared_examples.rb rename to spec/support/shared_examples/lib/gitlab/usage_data_counters/usage_counter_shared_examples.rb -- GitLab From 22c0c91c7b8c9dfae0f973a2306c9ee4d1bd19ea Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 10 Mar 2022 14:51:03 +0100 Subject: [PATCH 2/4] Remove dependency_list usage counter This counter counting usage by project, resulting in an ever growing set in Redis as the feature gains adoption. This data wasn't useful, while still consuming a relatively high amount of memory, so we removed it. A better approach would be using Redis HLL counters for this. --- .../controllers/projects/dependencies_controller.rb | 2 -- .../20210216175211_dependency_list_usages_total.yml | 2 +- ee/lib/ee/gitlab/usage_data.rb | 1 - ee/lib/gitlab/usage_counters/dependency_list.rb | 11 ----------- .../projects/dependencies_controller_spec.rb | 10 ---------- ee/spec/lib/ee/gitlab/usage_data_spec.rb | 1 - .../lib/gitlab/usage_counters/dependency_list_spec.rb | 7 ------- 7 files changed, 1 insertion(+), 33 deletions(-) delete mode 100644 ee/lib/gitlab/usage_counters/dependency_list.rb delete mode 100644 ee/spec/lib/gitlab/usage_counters/dependency_list_spec.rb diff --git a/ee/app/controllers/projects/dependencies_controller.rb b/ee/app/controllers/projects/dependencies_controller.rb index 9d4294524323ec..d634e6ed1adb49 100644 --- a/ee/app/controllers/projects/dependencies_controller.rb +++ b/ee/app/controllers/projects/dependencies_controller.rb @@ -18,8 +18,6 @@ def index render status: :ok end format.json do - ::Gitlab::UsageCounters::DependencyList.increment(project.id) - render json: serializer.represent(dependencies, build: report_service.build) end end diff --git a/ee/config/metrics/counts_all/20210216175211_dependency_list_usages_total.yml b/ee/config/metrics/counts_all/20210216175211_dependency_list_usages_total.yml index a4e877dd099b03..8001ce6aafea86 100644 --- a/ee/config/metrics/counts_all/20210216175211_dependency_list_usages_total.yml +++ b/ee/config/metrics/counts_all/20210216175211_dependency_list_usages_total.yml @@ -7,7 +7,7 @@ product_stage: secure product_group: group::composition analysis product_category: dependency_scanning value_type: number -status: active +status: removed time_frame: all data_source: redis distribution: diff --git a/ee/lib/ee/gitlab/usage_data.rb b/ee/lib/ee/gitlab/usage_data.rb index c68fece8a88ada..2d793cff2e3888 100644 --- a/ee/lib/ee/gitlab/usage_data.rb +++ b/ee/lib/ee/gitlab/usage_data.rb @@ -201,7 +201,6 @@ def system_usage_data usage_data[:counts].merge!( { confidential_epics: count(::Epic.confidential), - dependency_list_usages_total: redis_usage_data { ::Gitlab::UsageCounters::DependencyList.usage_totals[:total] }, epics: count(::Epic), epic_issues: count(::EpicIssue), geo_nodes: count(::GeoNode), diff --git a/ee/lib/gitlab/usage_counters/dependency_list.rb b/ee/lib/gitlab/usage_counters/dependency_list.rb deleted file mode 100644 index da897639618d40..00000000000000 --- a/ee/lib/gitlab/usage_counters/dependency_list.rb +++ /dev/null @@ -1,11 +0,0 @@ -# frozen_string_literal: true - -module Gitlab - module UsageCounters - class DependencyList < Common - def self.base_key - 'DEPENDENCY_LIST_USAGE_COUNTER' - end - end - end -end diff --git a/ee/spec/controllers/projects/dependencies_controller_spec.rb b/ee/spec/controllers/projects/dependencies_controller_spec.rb index b2083605ee05d1..9337d1f1fd28c6 100644 --- a/ee/spec/controllers/projects/dependencies_controller_spec.rb +++ b/ee/spec/controllers/projects/dependencies_controller_spec.rb @@ -43,16 +43,6 @@ end end - context 'when usage ping is collected' do - let(:user) { developer } - - it 'counts usage of the feature' do - expect(::Gitlab::UsageCounters::DependencyList).to receive(:increment).with(project.id) - - get :index, params: params, format: :json - end - end - context 'with existing report' do let_it_be(:pipeline) { create(:ee_ci_pipeline, :with_dependency_list_report, project: project) } diff --git a/ee/spec/lib/ee/gitlab/usage_data_spec.rb b/ee/spec/lib/ee/gitlab/usage_data_spec.rb index 418556fd03a462..70aece52b280db 100644 --- a/ee/spec/lib/ee/gitlab/usage_data_spec.rb +++ b/ee/spec/lib/ee/gitlab/usage_data_spec.rb @@ -93,7 +93,6 @@ container_scanning_jobs coverage_fuzzing_jobs dast_jobs - dependency_list_usages_total dependency_scanning_jobs epics epics_deepest_relationship_level diff --git a/ee/spec/lib/gitlab/usage_counters/dependency_list_spec.rb b/ee/spec/lib/gitlab/usage_counters/dependency_list_spec.rb deleted file mode 100644 index 66c6fc3531fe99..00000000000000 --- a/ee/spec/lib/gitlab/usage_counters/dependency_list_spec.rb +++ /dev/null @@ -1,7 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Gitlab::UsageCounters::DependencyList, :clean_gitlab_redis_shared_state do - it_behaves_like 'a usage counter' -end -- GitLab From 91b4a64615b667331cc1ee8407e623c2e520103e Mon Sep 17 00:00:00 2001 From: Bob Van Landuyt Date: Thu, 10 Mar 2022 15:51:01 +0100 Subject: [PATCH 3/4] Remove dependency list usage data from Redis This removes any existing counters for the dependency list usage data from Redis. Freeing up the memory it was taking. Changelog: other EE: true --- ...e_dependency_list_usage_data_from_redis.rb | 13 +++++++++++ db/schema_migrations/20220310141349 | 1 + ...endency_list_usage_data_from_redis_spec.rb | 23 +++++++++++++++++++ 3 files changed, 37 insertions(+) create mode 100644 db/post_migrate/20220310141349_remove_dependency_list_usage_data_from_redis.rb create mode 100644 db/schema_migrations/20220310141349 create mode 100644 spec/migrations/20220310141349_remove_dependency_list_usage_data_from_redis_spec.rb diff --git a/db/post_migrate/20220310141349_remove_dependency_list_usage_data_from_redis.rb b/db/post_migrate/20220310141349_remove_dependency_list_usage_data_from_redis.rb new file mode 100644 index 00000000000000..3c1e6714529173 --- /dev/null +++ b/db/post_migrate/20220310141349_remove_dependency_list_usage_data_from_redis.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +class RemoveDependencyListUsageDataFromRedis < Gitlab::Database::Migration[1.0] + disable_ddl_transaction! + + def up + Gitlab::Redis::SharedState.with { |r| r.del("DEPENDENCY_LIST_USAGE_COUNTER") } + end + + def down + # no-op + end +end diff --git a/db/schema_migrations/20220310141349 b/db/schema_migrations/20220310141349 new file mode 100644 index 00000000000000..d52b2d997a4df3 --- /dev/null +++ b/db/schema_migrations/20220310141349 @@ -0,0 +1 @@ +39785d4140c7345ddbe62417576381654ce22d505ee5c92a84425f0a3f8e4935 \ No newline at end of file diff --git a/spec/migrations/20220310141349_remove_dependency_list_usage_data_from_redis_spec.rb b/spec/migrations/20220310141349_remove_dependency_list_usage_data_from_redis_spec.rb new file mode 100644 index 00000000000000..c00685c1397776 --- /dev/null +++ b/spec/migrations/20220310141349_remove_dependency_list_usage_data_from_redis_spec.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +require 'spec_helper' +require_migration! + +RSpec.describe RemoveDependencyListUsageDataFromRedis, :migration, :clean_gitlab_redis_shared_state do + let(:key) { "DEPENDENCY_LIST_USAGE_COUNTER" } + + describe "#up" do + it 'removes the hash from redis' do + with_redis do |redis| + redis.hincrby(key, 1, 1) + redis.hincrby(key, 2, 1) + end + + expect { migrate! }.to change { with_redis { |r| r.hgetall(key) } }.from({ '1' => '1', '2' => '1' }).to({}) + end + end + + def with_redis(&block) + Gitlab::Redis::SharedState.with(&block) + end +end -- GitLab From 47d21186242e3bd913befd3d4fa367ee4f73d9d6 Mon Sep 17 00:00:00 2001 From: Alina Mihaila Date: Fri, 11 Mar 2022 16:14:19 +0000 Subject: [PATCH 4/4] Mention milestone where count was removed --- .../counts_all/20210216175211_dependency_list_usages_total.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/ee/config/metrics/counts_all/20210216175211_dependency_list_usages_total.yml b/ee/config/metrics/counts_all/20210216175211_dependency_list_usages_total.yml index 8001ce6aafea86..042defc4ac0f14 100644 --- a/ee/config/metrics/counts_all/20210216175211_dependency_list_usages_total.yml +++ b/ee/config/metrics/counts_all/20210216175211_dependency_list_usages_total.yml @@ -8,6 +8,7 @@ product_group: group::composition analysis product_category: dependency_scanning value_type: number status: removed +milestone_removed: "14.9" time_frame: all data_source: redis distribution: -- GitLab