diff --git a/Gemfile b/Gemfile index 282c503d3c8adc222e9092983cd46e72dc5f025e..70aa3cc993221606b22c8220a5e8b713f8030c47 100644 --- a/Gemfile +++ b/Gemfile @@ -479,6 +479,7 @@ gem 'flipper', '~> 0.17.1' gem 'flipper-active_record', '~> 0.17.1' gem 'flipper-active_support_cache_store', '~> 0.17.1' gem 'unleash', '~> 0.1.5' +gem 'gitlab-experiment', '~> 0.4.2' # Structured logging gem 'lograge', '~> 0.5' diff --git a/Gemfile.lock b/Gemfile.lock index 22ebab2492c449f7b98c0314266acd0db2b7591a..14d110666b262a53d51bf869da2d21d463a63c46 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -427,6 +427,9 @@ GEM github-markup (1.7.0) gitlab-chronic (0.10.5) numerizer (~> 0.2) + gitlab-experiment (0.4.2) + activesupport (>= 3.0) + scientist (~> 1.5, >= 1.5.0) gitlab-fog-azure-rm (1.0.0) azure-storage-blob (~> 2.0) azure-storage-common (~> 2.0) @@ -1087,6 +1090,7 @@ GEM sawyer (0.8.2) addressable (>= 2.3.5) faraday (> 0.8, < 2.0) + scientist (1.5.0) scss_lint (0.59.0) sass (~> 3.5, >= 3.5.5) securecompare (1.0.0) @@ -1353,6 +1357,7 @@ DEPENDENCIES gitaly (~> 13.7.0.pre.rc1) github-markup (~> 1.7.0) gitlab-chronic (~> 0.10.5) + gitlab-experiment (~> 0.4.2) gitlab-fog-azure-rm (~> 1.0) gitlab-labkit (= 0.13.3) gitlab-license (~> 1.0) diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb index b2b0c423da2c0ec9637cbc8985e14d6fa78ed299..552905ca5225661fcd374a0821715be95c2c9cc8 100644 --- a/app/controllers/projects/issues_controller.rb +++ b/app/controllers/projects/issues_controller.rb @@ -59,6 +59,10 @@ class Projects::IssuesController < Projects::ApplicationController around_action :allow_gitaly_ref_name_caching, only: [:discussions] + before_action :run_null_hypothesis_experiment, + only: [:index, :new, :create], + if: -> { Feature.enabled?(:gitlab_experiments) } + respond_to :html alias_method :designs, :show @@ -390,6 +394,14 @@ def service_desk? action_name == 'service_desk' end + def run_null_hypothesis_experiment + experiment(:null_hypothesis, project: project) do |e| + e.use { } # define the control + e.try { } # define the candidate + e.track(action_name) # track the action so we can build a funnel + end + end + # Overridden in EE def create_vulnerability_issue_link(issue); end end diff --git a/changelogs/unreleased/jj-add-gitlab-experiment.yml b/changelogs/unreleased/jj-add-gitlab-experiment.yml new file mode 100644 index 0000000000000000000000000000000000000000..99b8026d61ad66dd473b054549a30623e7dd765d --- /dev/null +++ b/changelogs/unreleased/jj-add-gitlab-experiment.yml @@ -0,0 +1,5 @@ +--- +title: Add the gitlab-experiment gem, with configuration +merge_request: 45840 +author: +type: added diff --git a/config/feature_flags/development/gitlab_experiments.yml b/config/feature_flags/development/gitlab_experiments.yml new file mode 100644 index 0000000000000000000000000000000000000000..51fa6aa4529ba16946799d5a086510af02de5230 --- /dev/null +++ b/config/feature_flags/development/gitlab_experiments.yml @@ -0,0 +1,8 @@ +--- +name: gitlab_experiments +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45840 +rollout_issue_url: +milestone: '13.7' +type: development +group: group::adoption +default_enabled: false diff --git a/config/feature_flags/experiment/null_hypothesis.yml b/config/feature_flags/experiment/null_hypothesis.yml new file mode 100644 index 0000000000000000000000000000000000000000..716b0711ef106c5853c99b6e7167e71e08af28ab --- /dev/null +++ b/config/feature_flags/experiment/null_hypothesis.yml @@ -0,0 +1,7 @@ +--- +name: null_hypothesis +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45840 +rollout_issue_url: +type: experiment +group: group::adoption +default_enabled: false diff --git a/config/initializers/gitlab_experiment.rb b/config/initializers/gitlab_experiment.rb new file mode 100644 index 0000000000000000000000000000000000000000..a09a447b65fd101c54345bfde516197cd0192e87 --- /dev/null +++ b/config/initializers/gitlab_experiment.rb @@ -0,0 +1,47 @@ +# frozen_string_literal: true + +Gitlab::Experiment.configure do |config| + # Logic this project uses to resolve a variant for a given experiment. + # + # This can return an instance of any object that responds to `name`, or can + # return a variant name as a symbol or string. + # + # This block will be executed within the scope of the experiment instance, + # so can easily access experiment methods, like getting the name or context. + config.variant_resolver = lambda do |requested_variant| + # Return the variant if one was requested in code: + break requested_variant if requested_variant.present? + + # Use Feature interface to determine the variant by passing the experiment, + # which responds to `flipper_id` and `session_id` to accommodate adapters. + variant_names.first if Feature.enabled?(name, self, type: :experiment) + end + + # Tracking behavior can be implemented to link an event to an experiment. + # + # Similar to the variant_resolver, this is called within the scope of the + # experiment instance and so can access any methods on the experiment, + # such as name and signature. + config.tracking_behavior = lambda do |event, args| + Gitlab::Tracking.event(name, event.to_s, **args.merge( + context: (args[:context] || []) << SnowplowTracker::SelfDescribingJson.new( + 'iglu:com.gitlab/gitlab_experiment/jsonschema/0-3-0', signature + ) + )) + end + + # Called at the end of every experiment run, with the result. + # + # You may want to track that you've assigned a variant to a given context, + # or push the experiment into the client or publish results elsewhere, like + # into redis or postgres. Also called within the scope of the experiment + # instance. + config.publishing_behavior = lambda do |result| + # Track the event using our own configured tracking logic. + track(:assignment) + + # Push the experiment knowledge into the front end. The signature contains + # the context key, and the variant that has been determined. + Gon.push({ experiment: { name => signature } }, true) + end +end diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb index 1fcbc8fa1730ce18c20fb92d4143be5cf63386fe..17dfe26bd8276f6d096474050c43aae1cc205f9e 100644 --- a/lib/feature/shared.rb +++ b/lib/feature/shared.rb @@ -23,7 +23,7 @@ module Shared example: <<-EOS Feature.enabled?(:my_feature_flag, project) Feature.enabled?(:my_feature_flag, project, type: :development) - push_frontend_feature_flag?(:my_feature_flag, project) + push_frontend_feature_flag(:my_feature_flag, project) EOS }, ops: { @@ -33,8 +33,8 @@ module Shared ee_only: false, default_enabled: false, example: <<-EOS - Feature.enabled?(:my_ops_flag, type: ops) - push_frontend_feature_flag?(:my_ops_flag, project, type: :ops) + Feature.enabled?(:my_ops_flag, type: :ops) + push_frontend_feature_flag(:my_ops_flag, project, type: :ops) EOS }, licensed: { @@ -48,6 +48,16 @@ module Shared project.feature_available?(:my_licensed_feature) namespace.feature_available?(:my_licensed_feature) EOS + }, + experiment: { + description: 'Short lived, used specifically to run A/B/n experiments.', + optional: true, + rollout_issue: true, + ee_only: true, + default_enabled: false, + example: <<-EOS + experiment(:my_experiment, project: project, actor: current_user) { ...variant code... } + EOS } }.freeze diff --git a/spec/controllers/projects/issues_controller_spec.rb b/spec/controllers/projects/issues_controller_spec.rb index 4ad5e649522924154a7591f11a46326619ab6531..12c8c84dd77c19306f238e1ec201616fbbd41b1f 100644 --- a/spec/controllers/projects/issues_controller_spec.rb +++ b/spec/controllers/projects/issues_controller_spec.rb @@ -62,6 +62,56 @@ expect(response).to have_gitlab_http_status(:moved_permanently) end end + + describe 'the null hypothesis experiment', :snowplow do + it 'defines the expected before actions' do + expect(controller).to use_before_action(:run_null_hypothesis_experiment) + end + + context 'when rolled out to 100%' do + it 'assigns the candidate experience and tracks the event' do + get :index, params: { namespace_id: project.namespace, project_id: project } + + expect_snowplow_event( + category: 'null_hypothesis', + action: 'index', + context: [{ + schema: 'iglu:com.gitlab/gitlab_experiment/jsonschema/0-3-0', + data: { variant: 'candidate', experiment: 'null_hypothesis', key: anything } + }] + ) + end + end + + context 'when not rolled out' do + before do + stub_feature_flags(null_hypothesis: false) + end + + it 'assigns the control experience and tracks the event' do + get :index, params: { namespace_id: project.namespace, project_id: project } + + expect_snowplow_event( + category: 'null_hypothesis', + action: 'index', + context: [{ + schema: 'iglu:com.gitlab/gitlab_experiment/jsonschema/0-3-0', + data: { variant: 'control', experiment: 'null_hypothesis', key: anything } + }] + ) + end + end + + context 'when gitlab_experiments is disabled' do + it 'does not run the experiment at all' do + stub_feature_flags(gitlab_experiments: false) + + expect(controller).not_to receive(:run_null_hypothesis_experiment) + + get :index, params: { namespace_id: project.namespace, project_id: project } + end + end + end end context 'internal issue tracker' do diff --git a/spec/support/helpers/snowplow_helpers.rb b/spec/support/helpers/snowplow_helpers.rb index 15eac1b24fc589bb7cb0ddac2838a8c6f323ee01..70a4eadd8dec04652d1e788fbf88838f9db6b09b 100644 --- a/spec/support/helpers/snowplow_helpers.rb +++ b/spec/support/helpers/snowplow_helpers.rb @@ -31,7 +31,31 @@ module SnowplowHelpers # ) # end # end - def expect_snowplow_event(category:, action:, **kwargs) + # + # Passing context: + # + # Simply provide a hash that has the schema and data expected. + # + # expect_snowplow_event( + # category: 'Experiment', + # action: 'created', + # context: [ + # { + # schema: 'iglu:com.gitlab/.../0-3-0', + # data: { key: 'value' } + # } + # ] + # ) + def expect_snowplow_event(category:, action:, context: nil, **kwargs) + if context + kwargs[:context] = [] + context.each do |c| + expect(SnowplowTracker::SelfDescribingJson).to have_received(:new) + .with(c[:schema], c[:data]).at_least(:once) + kwargs[:context] << an_instance_of(SnowplowTracker::SelfDescribingJson) + end + end + expect(Gitlab::Tracking).to have_received(:event) # rubocop:disable RSpec/ExpectGitlabTracking .with(category, action, **kwargs).at_least(:once) end diff --git a/spec/support/snowplow.rb b/spec/support/snowplow.rb index 9635b1bd6569b64ed1b431db92b9fce212b9e1fe..0d6102f17050d995bc49cf865c362c7bd9b01e9a 100644 --- a/spec/support/snowplow.rb +++ b/spec/support/snowplow.rb @@ -17,6 +17,7 @@ stub_application_setting(snowplow_enabled: true) + allow(SnowplowTracker::SelfDescribingJson).to receive(:new).and_call_original allow(Gitlab::Tracking).to receive(:event).and_call_original # rubocop:disable RSpec/ExpectGitlabTracking end