From 52ea0d3d9cbde9f3f09dbb2c193e75c2b85e59c5 Mon Sep 17 00:00:00 2001
From: jejacks0n <jjackson@gitlab.com>
Date: Tue, 27 Oct 2020 12:44:36 -0600
Subject: [PATCH 1/6] Add new experiment feature flag type

- To be used by the experiment library configuration and to identify
differences between feature flag types.
- Fixes minor incorrectness in usage examples.
---
 lib/feature/shared.rb | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb
index 1fcbc8fa1730ce..10cd908eb03388 100644
--- a/lib/feature/shared.rb
+++ b/lib/feature/shared.rb
@@ -23,7 +23,7 @@ module Shared
         example: <<-EOS
           Feature.enabled?(:my_feature_flag, project)
           Feature.enabled?(:my_feature_flag, project, type: :development)
-          push_frontend_feature_flag?(:my_feature_flag, project)
+          push_frontend_feature_flag(:my_feature_flag, project)
         EOS
       },
       ops: {
@@ -33,8 +33,8 @@ module Shared
         ee_only: false,
         default_enabled: false,
         example: <<-EOS
-          Feature.enabled?(:my_ops_flag, type: ops)
-          push_frontend_feature_flag?(:my_ops_flag, project, type: :ops)
+          Feature.enabled?(:my_ops_flag, type: :ops)
+          push_frontend_feature_flag(:my_ops_flag, project, type: :ops)
         EOS
       },
       licensed: {
@@ -48,6 +48,17 @@ module Shared
           project.feature_available?(:my_licensed_feature)
           namespace.feature_available?(:my_licensed_feature)
         EOS
+      },
+      experiment: {
+        description: 'Experimental features for testing performance or engagement of an idea.',
+        optional: true,
+        rollout_issue: true,
+        ee_only: true,
+        default_enabled: false,
+        example: <<-EOS
+          experiment(:my_experiment, project: project, actor: current_user) { ...variant code... }
+          Gitlab::Experiment.run(:my_experiment, project: project) { ...variant code... }
+        EOS
       }
     }.freeze
 
-- 
GitLab


From cc54de6071fc3efe80b6d3947d79eb18b50ad05e Mon Sep 17 00:00:00 2001
From: jejacks0n <jjackson@gitlab.com>
Date: Tue, 27 Oct 2020 12:45:24 -0600
Subject: [PATCH 2/6] Add gitlab_experiment gem, and configure it

- This leaves the configuration comments intacts to help understanding.
---
 Gemfile                                       |  1 +
 Gemfile.lock                                  |  5 ++
 .../unreleased/jj-add-gitlab-experiment.yml   |  5 ++
 config/initializers/gitlab_experiment.rb      | 47 +++++++++++++++++++
 4 files changed, 58 insertions(+)
 create mode 100644 changelogs/unreleased/jj-add-gitlab-experiment.yml
 create mode 100644 config/initializers/gitlab_experiment.rb

diff --git a/Gemfile b/Gemfile
index 282c503d3c8adc..70aa3cc9932216 100644
--- a/Gemfile
+++ b/Gemfile
@@ -479,6 +479,7 @@ gem 'flipper', '~> 0.17.1'
 gem 'flipper-active_record', '~> 0.17.1'
 gem 'flipper-active_support_cache_store', '~> 0.17.1'
 gem 'unleash', '~> 0.1.5'
+gem 'gitlab-experiment', '~> 0.4.2'
 
 # Structured logging
 gem 'lograge', '~> 0.5'
diff --git a/Gemfile.lock b/Gemfile.lock
index 22ebab2492c449..14d110666b262a 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -427,6 +427,9 @@ GEM
     github-markup (1.7.0)
     gitlab-chronic (0.10.5)
       numerizer (~> 0.2)
+    gitlab-experiment (0.4.2)
+      activesupport (>= 3.0)
+      scientist (~> 1.5, >= 1.5.0)
     gitlab-fog-azure-rm (1.0.0)
       azure-storage-blob (~> 2.0)
       azure-storage-common (~> 2.0)
@@ -1087,6 +1090,7 @@ GEM
     sawyer (0.8.2)
       addressable (>= 2.3.5)
       faraday (> 0.8, < 2.0)
+    scientist (1.5.0)
     scss_lint (0.59.0)
       sass (~> 3.5, >= 3.5.5)
     securecompare (1.0.0)
@@ -1353,6 +1357,7 @@ DEPENDENCIES
   gitaly (~> 13.7.0.pre.rc1)
   github-markup (~> 1.7.0)
   gitlab-chronic (~> 0.10.5)
+  gitlab-experiment (~> 0.4.2)
   gitlab-fog-azure-rm (~> 1.0)
   gitlab-labkit (= 0.13.3)
   gitlab-license (~> 1.0)
diff --git a/changelogs/unreleased/jj-add-gitlab-experiment.yml b/changelogs/unreleased/jj-add-gitlab-experiment.yml
new file mode 100644
index 00000000000000..99b8026d61ad66
--- /dev/null
+++ b/changelogs/unreleased/jj-add-gitlab-experiment.yml
@@ -0,0 +1,5 @@
+---
+title: Add the gitlab-experiment gem, with configuration
+merge_request: 45840
+author:
+type: added
diff --git a/config/initializers/gitlab_experiment.rb b/config/initializers/gitlab_experiment.rb
new file mode 100644
index 00000000000000..a09a447b65fd10
--- /dev/null
+++ b/config/initializers/gitlab_experiment.rb
@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+Gitlab::Experiment.configure do |config|
+  # Logic this project uses to resolve a variant for a given experiment.
+  #
+  # This can return an instance of any object that responds to `name`, or can
+  # return a variant name as a symbol or string.
+  #
+  # This block will be executed within the scope of the experiment instance,
+  # so can easily access experiment methods, like getting the name or context.
+  config.variant_resolver = lambda do |requested_variant|
+    # Return the variant if one was requested in code:
+    break requested_variant if requested_variant.present?
+
+    # Use Feature interface to determine the variant by passing the experiment,
+    # which responds to `flipper_id` and `session_id` to accommodate adapters.
+    variant_names.first if Feature.enabled?(name, self, type: :experiment)
+  end
+
+  # Tracking behavior can be implemented to link an event to an experiment.
+  #
+  # Similar to the variant_resolver, this is called within the scope of the
+  # experiment instance and so can access any methods on the experiment,
+  # such as name and signature.
+  config.tracking_behavior = lambda do |event, args|
+    Gitlab::Tracking.event(name, event.to_s, **args.merge(
+      context: (args[:context] || []) << SnowplowTracker::SelfDescribingJson.new(
+        'iglu:com.gitlab/gitlab_experiment/jsonschema/0-3-0', signature
+      )
+    ))
+  end
+
+  # Called at the end of every experiment run, with the result.
+  #
+  # You may want to track that you've assigned a variant to a given context,
+  # or push the experiment into the client or publish results elsewhere, like
+  # into redis or postgres. Also called within the scope of the experiment
+  # instance.
+  config.publishing_behavior = lambda do |result|
+    # Track the event using our own configured tracking logic.
+    track(:assignment)
+
+    # Push the experiment knowledge into the front end. The signature contains
+    # the context key, and the variant that has been determined.
+    Gon.push({ experiment: { name => signature } }, true)
+  end
+end
-- 
GitLab


From 6ea9d9e5509348ea5317b8af812fed98b63de9bc Mon Sep 17 00:00:00 2001
From: jejacks0n <jjackson@gitlab.com>
Date: Tue, 27 Oct 2020 13:15:05 -0600
Subject: [PATCH 3/6] Adds a null hypothesis (or A/A) experiment
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

- This is about vetting the gitlab_experiment gem and generating some
data that we’ll be able to process and build a funnel from.
---
 app/controllers/projects/issues_controller.rb       | 10 ++++++++++
 config/feature_flags/experiment/null_hypothesis.yml |  7 +++++++
 spec/controllers/projects/issues_controller_spec.rb |  7 +++++++
 3 files changed, 24 insertions(+)
 create mode 100644 config/feature_flags/experiment/null_hypothesis.yml

diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb
index b2b0c423da2c0e..ac9566cd374b9a 100644
--- a/app/controllers/projects/issues_controller.rb
+++ b/app/controllers/projects/issues_controller.rb
@@ -59,6 +59,8 @@ class Projects::IssuesController < Projects::ApplicationController
 
   around_action :allow_gitaly_ref_name_caching, only: [:discussions]
 
+  before_action :run_null_hypothesis_experiment, only: [:index, :new, :create]
+
   respond_to :html
 
   alias_method :designs, :show
@@ -390,6 +392,14 @@ def service_desk?
     action_name == 'service_desk'
   end
 
+  def run_null_hypothesis_experiment
+    experiment(:null_hypothesis, project: project) do |e|
+      e.use { } # define the control
+      e.try { } # define the candidate
+      e.track(action_name) # track the action so we can build a funnel
+    end
+  end
+
   # Overridden in EE
   def create_vulnerability_issue_link(issue); end
 end
diff --git a/config/feature_flags/experiment/null_hypothesis.yml b/config/feature_flags/experiment/null_hypothesis.yml
new file mode 100644
index 00000000000000..716b0711ef106c
--- /dev/null
+++ b/config/feature_flags/experiment/null_hypothesis.yml
@@ -0,0 +1,7 @@
+---
+name: null_hypothesis
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45840
+rollout_issue_url:
+type: experiment
+group: group::adoption
+default_enabled: false
diff --git a/spec/controllers/projects/issues_controller_spec.rb b/spec/controllers/projects/issues_controller_spec.rb
index 4ad5e649522924..fbe186e410f788 100644
--- a/spec/controllers/projects/issues_controller_spec.rb
+++ b/spec/controllers/projects/issues_controller_spec.rb
@@ -62,6 +62,13 @@
           expect(response).to have_gitlab_http_status(:moved_permanently)
         end
       end
+
+      it 'runs and tracks the null hypothesis experiment' do
+        expect(experiment = Gitlab::Experiment.new(:double)).to receive(:track).with('index')
+        expect(controller).to receive(:experiment).with(:null_hypothesis, project: project).and_yield(experiment)
+
+        get :index, params: { namespace_id: project.namespace, project_id: project }
+      end
     end
 
     context 'internal issue tracker' do
-- 
GitLab


From 9550c96b93971533ce03b3b4575b1ab57f47af55 Mon Sep 17 00:00:00 2001
From: jejacks0n <jjackson@gitlab.com>
Date: Thu, 19 Nov 2020 16:12:53 -0700
Subject: [PATCH 4/6] Update description of experiment feature type

---
 lib/feature/shared.rb | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/lib/feature/shared.rb b/lib/feature/shared.rb
index 10cd908eb03388..17dfe26bd8276f 100644
--- a/lib/feature/shared.rb
+++ b/lib/feature/shared.rb
@@ -50,14 +50,13 @@ module Shared
         EOS
       },
       experiment: {
-        description: 'Experimental features for testing performance or engagement of an idea.',
+        description: 'Short lived, used specifically to run A/B/n experiments.',
         optional: true,
         rollout_issue: true,
         ee_only: true,
         default_enabled: false,
         example: <<-EOS
           experiment(:my_experiment, project: project, actor: current_user) { ...variant code... }
-          Gitlab::Experiment.run(:my_experiment, project: project) { ...variant code... }
         EOS
       }
     }.freeze
-- 
GitLab


From 20eace10786f89b7ee144b148d2d8b73a6d2d6c1 Mon Sep 17 00:00:00 2001
From: jejacks0n <jjackson@gitlab.com>
Date: Mon, 7 Dec 2020 15:16:16 -0700
Subject: [PATCH 5/6] Wrap the library usage within a feature flag

- This just keeps anyone from running into any of these code paths while
not on gitlab.com as we review and refine the experiment.
---
 app/controllers/projects/issues_controller.rb           | 4 +++-
 config/feature_flags/development/gitlab_experiments.yml | 8 ++++++++
 spec/controllers/projects/issues_controller_spec.rb     | 8 +++++++-
 3 files changed, 18 insertions(+), 2 deletions(-)
 create mode 100644 config/feature_flags/development/gitlab_experiments.yml

diff --git a/app/controllers/projects/issues_controller.rb b/app/controllers/projects/issues_controller.rb
index ac9566cd374b9a..552905ca522566 100644
--- a/app/controllers/projects/issues_controller.rb
+++ b/app/controllers/projects/issues_controller.rb
@@ -59,7 +59,9 @@ class Projects::IssuesController < Projects::ApplicationController
 
   around_action :allow_gitaly_ref_name_caching, only: [:discussions]
 
-  before_action :run_null_hypothesis_experiment, only: [:index, :new, :create]
+  before_action :run_null_hypothesis_experiment,
+                only: [:index, :new, :create],
+                if: -> { Feature.enabled?(:gitlab_experiments) }
 
   respond_to :html
 
diff --git a/config/feature_flags/development/gitlab_experiments.yml b/config/feature_flags/development/gitlab_experiments.yml
new file mode 100644
index 00000000000000..51fa6aa4529ba1
--- /dev/null
+++ b/config/feature_flags/development/gitlab_experiments.yml
@@ -0,0 +1,8 @@
+---
+name: gitlab_experiments
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/45840
+rollout_issue_url: 
+milestone: '13.7'
+type: development
+group: group::adoption
+default_enabled: false
diff --git a/spec/controllers/projects/issues_controller_spec.rb b/spec/controllers/projects/issues_controller_spec.rb
index fbe186e410f788..6a1fb920f06dc9 100644
--- a/spec/controllers/projects/issues_controller_spec.rb
+++ b/spec/controllers/projects/issues_controller_spec.rb
@@ -64,8 +64,14 @@
       end
 
       it 'runs and tracks the null hypothesis experiment' do
+        stub_feature_flags(gitlab_experiments: false)
+
+        get :index, params: { namespace_id: project.namespace, project_id: project }
+
+        stub_feature_flags(gitlab_experiments: true)
+
         expect(experiment = Gitlab::Experiment.new(:double)).to receive(:track).with('index')
-        expect(controller).to receive(:experiment).with(:null_hypothesis, project: project).and_yield(experiment)
+        expect(controller).to receive(:experiment).once.with(:null_hypothesis, project: project).and_yield(experiment)
 
         get :index, params: { namespace_id: project.namespace, project_id: project }
       end
-- 
GitLab


From 5e2fe18c2e669300026c3de4d49ae867f410b2fe Mon Sep 17 00:00:00 2001
From: jejacks0n <jjackson@gitlab.com>
Date: Wed, 9 Dec 2020 15:49:44 -0700
Subject: [PATCH 6/6] Improve the specs to cover rollout

---
 .../projects/issues_controller_spec.rb        | 51 ++++++++++++++++---
 spec/support/helpers/snowplow_helpers.rb      | 26 +++++++++-
 spec/support/snowplow.rb                      |  1 +
 3 files changed, 70 insertions(+), 8 deletions(-)

diff --git a/spec/controllers/projects/issues_controller_spec.rb b/spec/controllers/projects/issues_controller_spec.rb
index 6a1fb920f06dc9..12c8c84dd77c19 100644
--- a/spec/controllers/projects/issues_controller_spec.rb
+++ b/spec/controllers/projects/issues_controller_spec.rb
@@ -63,17 +63,54 @@
         end
       end
 
-      it 'runs and tracks the null hypothesis experiment' do
-        stub_feature_flags(gitlab_experiments: false)
+      describe 'the null hypothesis experiment', :snowplow do
+        it 'defines the expected before actions' do
+          expect(controller).to use_before_action(:run_null_hypothesis_experiment)
+        end
 
-        get :index, params: { namespace_id: project.namespace, project_id: project }
+        context 'when rolled out to 100%' do
+          it 'assigns the candidate experience and tracks the event' do
+            get :index, params: { namespace_id: project.namespace, project_id: project }
+
+            expect_snowplow_event(
+              category: 'null_hypothesis',
+              action: 'index',
+              context: [{
+                schema: 'iglu:com.gitlab/gitlab_experiment/jsonschema/0-3-0',
+                data: { variant: 'candidate', experiment: 'null_hypothesis', key: anything }
+              }]
+            )
+          end
+        end
 
-        stub_feature_flags(gitlab_experiments: true)
+        context 'when not rolled out' do
+          before do
+            stub_feature_flags(null_hypothesis: false)
+          end
 
-        expect(experiment = Gitlab::Experiment.new(:double)).to receive(:track).with('index')
-        expect(controller).to receive(:experiment).once.with(:null_hypothesis, project: project).and_yield(experiment)
+          it 'assigns the control experience and tracks the event' do
+            get :index, params: { namespace_id: project.namespace, project_id: project }
+
+            expect_snowplow_event(
+              category: 'null_hypothesis',
+              action: 'index',
+              context: [{
+                schema: 'iglu:com.gitlab/gitlab_experiment/jsonschema/0-3-0',
+                data: { variant: 'control', experiment: 'null_hypothesis', key: anything }
+              }]
+            )
+          end
+        end
 
-        get :index, params: { namespace_id: project.namespace, project_id: project }
+        context 'when gitlab_experiments is disabled' do
+          it 'does not run the experiment at all' do
+            stub_feature_flags(gitlab_experiments: false)
+
+            expect(controller).not_to receive(:run_null_hypothesis_experiment)
+
+            get :index, params: { namespace_id: project.namespace, project_id: project }
+          end
+        end
       end
     end
 
diff --git a/spec/support/helpers/snowplow_helpers.rb b/spec/support/helpers/snowplow_helpers.rb
index 15eac1b24fc589..70a4eadd8dec04 100644
--- a/spec/support/helpers/snowplow_helpers.rb
+++ b/spec/support/helpers/snowplow_helpers.rb
@@ -31,7 +31,31 @@ module SnowplowHelpers
   #       )
   #     end
   #   end
-  def expect_snowplow_event(category:, action:, **kwargs)
+  #
+  # Passing context:
+  #
+  #   Simply provide a hash that has the schema and data expected.
+  #
+  #   expect_snowplow_event(
+  #     category: 'Experiment',
+  #     action: 'created',
+  #     context: [
+  #       {
+  #         schema: 'iglu:com.gitlab/.../0-3-0',
+  #         data: { key: 'value' }
+  #       }
+  #     ]
+  #   )
+  def expect_snowplow_event(category:, action:, context: nil, **kwargs)
+    if context
+      kwargs[:context] = []
+      context.each do |c|
+        expect(SnowplowTracker::SelfDescribingJson).to have_received(:new)
+          .with(c[:schema], c[:data]).at_least(:once)
+        kwargs[:context] << an_instance_of(SnowplowTracker::SelfDescribingJson)
+      end
+    end
+
     expect(Gitlab::Tracking).to have_received(:event) # rubocop:disable RSpec/ExpectGitlabTracking
       .with(category, action, **kwargs).at_least(:once)
   end
diff --git a/spec/support/snowplow.rb b/spec/support/snowplow.rb
index 9635b1bd6569b6..0d6102f17050d9 100644
--- a/spec/support/snowplow.rb
+++ b/spec/support/snowplow.rb
@@ -17,6 +17,7 @@
 
     stub_application_setting(snowplow_enabled: true)
 
+    allow(SnowplowTracker::SelfDescribingJson).to receive(:new).and_call_original
     allow(Gitlab::Tracking).to receive(:event).and_call_original # rubocop:disable RSpec/ExpectGitlabTracking
   end
 
-- 
GitLab