From 04d5c69d8a239c1b290821ff23b5a54f7ce1cf57 Mon Sep 17 00:00:00 2001
From: "Balasankar \"Balu\" C" <balasankar@gitlab.com>
Date: Mon, 25 Oct 2021 12:33:00 +0530
Subject: [PATCH] Add knapsack for parallelizing tests not run on a cluster

Signed-off-by: Balasankar "Balu" C <balasankar@gitlab.com>
---
 .gitignore           |  1 +
 .gitlab-ci.yml       |  7 +++----
 Gemfile              | 23 +++++++++++++----------
 Gemfile.lock         |  3 +++
 Rakefile             |  2 ++
 scripts/ci/run_specs | 20 ++++++++++++++++++++
 6 files changed, 42 insertions(+), 14 deletions(-)
 create mode 100644 Rakefile
 create mode 100755 scripts/ci/run_specs

diff --git a/.gitignore b/.gitignore
index f2d41534c9..2208ad1abb 100644
--- a/.gitignore
+++ b/.gitignore
@@ -8,3 +8,4 @@ gems
 .bundle
 build
 spec/examples.txt
+knapsack_rspec_report.json
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c849ce60d0..edc39ee20c 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -575,10 +575,7 @@ review-docs-cleanup:
     S3_CONFIG_PATH: /etc/gitlab/minio
   script:
     - ./scripts/ci/install_spec_dependencies
-    - if [[ -n "${VARIABLES_FILE}" ]]; then source "${VARIABLES_FILE}"; ./scripts/ci/feature_spec_setup; else ./scripts/ci/integration_spec_setup; fi
-    - bundle config set path 'gems'
-    - bundle install -j $(nproc)
-    - bundle exec rspec -c -f d spec -t "${RSPEC_TAGS}"
+    - ./scripts/ci/run_specs
   artifacts:
     when: on_failure
     expire_in: 7d
@@ -654,6 +651,7 @@ specs_without_cluster:
   variables:
     RSPEC_TAGS: ~type:feature
   needs: []
+  parallel: 5
   except:
     refs:
       - /(^docs[\/-].+|.+-docs$)/
@@ -671,6 +669,7 @@ production_specs_without_cluster:
   extends: .production_specs
   variables:
     RSPEC_TAGS: ~type:feature
+  parallel: 5
   except:
     refs:
       - /(^docs[\/-].+|.+-docs$)/
diff --git a/Gemfile b/Gemfile
index d6f88cf9bc..41f5a60643 100644
--- a/Gemfile
+++ b/Gemfile
@@ -1,15 +1,18 @@
 source 'https://rubygems.org'
 
-gem 'rspec'
-gem 'rspec-retry'
-gem 'rspec-parameterized', require: false
-gem 'aws-sdk-s3'
-gem 'capybara'
-gem 'selenium-webdriver'
-gem 'capybara-screenshot'
-gem 'docker-api'
-gem 'rake'
-gem 'hash-deep-merge'
+group :test do
+  gem 'rspec'
+  gem 'rspec-retry'
+  gem 'rspec-parameterized', require: false
+  gem 'aws-sdk-s3'
+  gem 'capybara'
+  gem 'selenium-webdriver'
+  gem 'capybara-screenshot'
+  gem 'docker-api'
+  gem 'rake'
+  gem 'hash-deep-merge'
+  gem 'knapsack'
+end
 
 group :rubocop do
   gem 'gitlab-styles', '~> 6.1', require: false
diff --git a/Gemfile.lock b/Gemfile.lock
index c743950d12..7e8f7cb4c3 100644
--- a/Gemfile.lock
+++ b/Gemfile.lock
@@ -68,6 +68,8 @@ GEM
       concurrent-ruby (~> 1.0)
     ice_nine (0.11.2)
     jmespath (1.4.0)
+    knapsack (4.0.0)
+      rake
     launchy (2.4.3)
       addressable (~> 2.3)
     memoizable (0.4.2)
@@ -184,6 +186,7 @@ DEPENDENCIES
   docker-api
   gitlab-styles (~> 6.1)
   hash-deep-merge
+  knapsack
   pry
   rake
   rspec
diff --git a/Rakefile b/Rakefile
new file mode 100644
index 0000000000..c37d76c72c
--- /dev/null
+++ b/Rakefile
@@ -0,0 +1,2 @@
+require 'knapsack'
+Knapsack.load_tasks
diff --git a/scripts/ci/run_specs b/scripts/ci/run_specs
new file mode 100755
index 0000000000..dfb042042c
--- /dev/null
+++ b/scripts/ci/run_specs
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+if [[ -n "${VARIABLES_FILE}" ]]; then
+  source "${VARIABLES_FILE}"
+  ./scripts/ci/feature_spec_setup
+else
+  ./scripts/ci/integration_spec_setup
+fi
+
+bundle config set path 'gems'
+bundle install -j $(nproc)
+
+# For tests not being run on a cluster, use knapsack for parallelizing
+if [[ "${RSPEC_TAGS}" == "~type:feature" ]]; then
+  echo "Here"
+  echo "{}" > knapsack_rspec_report.json
+  bundle exec rake "knapsack:rspec[--color --format documentation --tag '${RSPEC_TAGS}']"
+else
+  bundle exec rspec -c -f d spec -t "${RSPEC_TAGS}"
+fi
-- 
GitLab