From 89b27d47504e2b908182cffc570129109862ea37 Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Mon, 18 Nov 2024 15:42:42 +1300 Subject: [PATCH 01/15] Adding custom parameters Added support for customer regstry by accepting login creds and custom registry link as env var Changelog: added --- lib/gitlab/qa/release.rb | 20 ++++++++++++++++++++ lib/gitlab/qa/runtime/env.rb | 12 ++++++++++++ 2 files changed, 32 insertions(+) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index 027dbf10..2ec170ad 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -59,6 +59,7 @@ module Gitlab DEFAULT_CANONICAL_TAG = 'nightly' DEV_REGISTRY = Gitlab::QA::Runtime::Env.qa_dev_registry COM_REGISTRY = Gitlab::QA::Runtime::Env.qa_com_registry + CUSTOM_REGISTRY = Gitlab::QA::Runtime::Env.qa_custom_registry InvalidImageNameError = Class.new(RuntimeError) @@ -110,12 +111,21 @@ module Gitlab end end + def assing_omnibus_mirror_fork_project + omnibus_project = image.match(CUSTOM_GITLAB_IMAGE_REGEX)[:project] + gitlab_project = "/registry.gitlab.com/gitlab-com/ops-sub-department/aws-gitlab-ai-integration/gitlab/" + + "#{image.gsub(omnibus_project, gitlab_project)}-qa" + end + def qa_image @qa_image ||= if omnibus_mirror? omnibus_project = image.match(CUSTOM_GITLAB_IMAGE_REGEX)[:project] gitlab_project = ci_project_path ? "/#{ci_project_path}/" : "/gitlab-org/gitlab/" "#{image.gsub(omnibus_project, gitlab_project)}-qa" + elsif omnibus_mirror_fork? + assing_omnibus_mirror_fork_project else "#{image}-qa" end @@ -162,6 +172,12 @@ module Gitlab } elsif omnibus_mirror? || omnibus_security? omnibus_login_params + elsif omnibus_mirror_fork? + { + username: Runtime::Env.gitlab_custom_username, + password: Runtime::Env.gitlab_custom_username_token, + registry: CUSTOM_REGISTRY + } end populate_registry_env_vars(params) @@ -203,6 +219,10 @@ module Gitlab image.start_with?("#{COM_REGISTRY}/gitlab-org/build/omnibus-gitlab-mirror/") end + def omnibus_mirror_fork? + image.start_with?("#{COM_REGISTRY}/gitlab-com/ops-sub-department/aws-gitlab-ai-integration/omnibus-gitlab-mirror") + end + def omnibus_security? image.start_with?("#{COM_REGISTRY}/gitlab-org/security/omnibus-gitlab/") end diff --git a/lib/gitlab/qa/runtime/env.rb b/lib/gitlab/qa/runtime/env.rb index 0bcaed00..92755824 100644 --- a/lib/gitlab/qa/runtime/env.rb +++ b/lib/gitlab/qa/runtime/env.rb @@ -219,6 +219,14 @@ module Gitlab env_var_value_if_defined('GITLAB_DEV_USERNAME') || 'gitlab-qa-bot' end + def gitlab_custom_username + env_var_value_if_defined('GITLAB_CUSTOM_USERNAME') + end + + def gitlab_custom_username_token + env_var_value_if_defined('GITLAB_CUSTOM_ACCESS_TOKEN') + end + def run_id @run_id ||= "gitlab-qa-run-#{Time.now.strftime('%Y-%m-%d-%H-%M-%S')}-#{SecureRandom.hex(4)}" end @@ -432,6 +440,10 @@ module Gitlab env_var_value_if_defined('QA_COM_REGISTRY') || 'registry.gitlab.com' end + def qa_custom_registry + env_var_value_if_defined('QA_CUSTOM_REGISTRY') || 'registry.gitlab.com' + end + def gitlab_license_mode env_var_value_if_defined('GITLAB_LICENSE_MODE') end -- GitLab From 3219c889c197721053a2bd26e0e36434652027be Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Mon, 18 Nov 2024 15:59:10 +1300 Subject: [PATCH 02/15] Removing and testing unwanted code --- lib/gitlab/qa/release.rb | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index 2ec170ad..23c4dcd7 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -111,12 +111,12 @@ module Gitlab end end - def assing_omnibus_mirror_fork_project - omnibus_project = image.match(CUSTOM_GITLAB_IMAGE_REGEX)[:project] - gitlab_project = "/registry.gitlab.com/gitlab-com/ops-sub-department/aws-gitlab-ai-integration/gitlab/" + # def assing_omnibus_mirror_fork_project + # omnibus_project = image.match(CUSTOM_GITLAB_IMAGE_REGEX)[:project] + # gitlab_project = "/registry.gitlab.com/gitlab-com/ops-sub-department/aws-gitlab-ai-integration/gitlab/" - "#{image.gsub(omnibus_project, gitlab_project)}-qa" - end + # "#{image.gsub(omnibus_project, gitlab_project)}-qa" + # end def qa_image @qa_image ||= if omnibus_mirror? @@ -124,8 +124,8 @@ module Gitlab gitlab_project = ci_project_path ? "/#{ci_project_path}/" : "/gitlab-org/gitlab/" "#{image.gsub(omnibus_project, gitlab_project)}-qa" - elsif omnibus_mirror_fork? - assing_omnibus_mirror_fork_project + # elsif omnibus_mirror_fork? + # assing_omnibus_mirror_fork_project else "#{image}-qa" end -- GitLab From 9048c5f12aa199f6e16815a8ec3ba42a81609951 Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Wed, 20 Nov 2024 08:40:22 +1300 Subject: [PATCH 03/15] Added regex for custome registry --- lib/gitlab/qa/release.rb | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index 23c4dcd7..7f81136d 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -25,6 +25,9 @@ module Gitlab \z }xi + CUSTOM_OMNIBUS_MIRROR_REGISTRY = %r{\A#{Regexp.escape(COM_REGISTRY)}/.*omnibus-gitlab-mirror\z} + + delegate :ci_project_path, to: Gitlab::QA::Runtime::Env # Official dev tag example: @@ -220,7 +223,7 @@ module Gitlab end def omnibus_mirror_fork? - image.start_with?("#{COM_REGISTRY}/gitlab-com/ops-sub-department/aws-gitlab-ai-integration/omnibus-gitlab-mirror") + image.start_with?(CUSTOM_OMNIBUS_MIRROR_REGISTRY) end def omnibus_security? -- GitLab From 7d27cc0d09a39b3fe5513d62128d99d0be256f78 Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Wed, 20 Nov 2024 08:41:53 +1300 Subject: [PATCH 04/15] Removed unwanted code --- lib/gitlab/qa/release.rb | 9 --------- 1 file changed, 9 deletions(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index 7f81136d..f80543e3 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -114,21 +114,12 @@ module Gitlab end end - # def assing_omnibus_mirror_fork_project - # omnibus_project = image.match(CUSTOM_GITLAB_IMAGE_REGEX)[:project] - # gitlab_project = "/registry.gitlab.com/gitlab-com/ops-sub-department/aws-gitlab-ai-integration/gitlab/" - - # "#{image.gsub(omnibus_project, gitlab_project)}-qa" - # end - def qa_image @qa_image ||= if omnibus_mirror? omnibus_project = image.match(CUSTOM_GITLAB_IMAGE_REGEX)[:project] gitlab_project = ci_project_path ? "/#{ci_project_path}/" : "/gitlab-org/gitlab/" "#{image.gsub(omnibus_project, gitlab_project)}-qa" - # elsif omnibus_mirror_fork? - # assing_omnibus_mirror_fork_project else "#{image}-qa" end -- GitLab From 90444374bcaf76442d00ffc9aa5152ad0e3181e4 Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Wed, 20 Nov 2024 09:02:47 +1300 Subject: [PATCH 05/15] Removing unintended new line --- lib/gitlab/qa/component/specs.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/gitlab/qa/component/specs.rb b/lib/gitlab/qa/component/specs.rb index 2cc7e842..677968f4 100644 --- a/lib/gitlab/qa/component/specs.rb +++ b/lib/gitlab/qa/component/specs.rb @@ -149,7 +149,6 @@ module Gitlab def docker_pull_qa_image_if_needed @docker.login(**release.login_params) if release.login_params - @docker.pull(image: qa_image) unless Runtime::Env.skip_pull? end -- GitLab From 4dc75737f6fda20cf750a9bfc72a22cb59cca8ba Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Wed, 20 Nov 2024 11:11:36 +1300 Subject: [PATCH 06/15] Removing extra newlines --- lib/gitlab/qa/release.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index f80543e3..7b8f2b63 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -27,7 +27,6 @@ module Gitlab CUSTOM_OMNIBUS_MIRROR_REGISTRY = %r{\A#{Regexp.escape(COM_REGISTRY)}/.*omnibus-gitlab-mirror\z} - delegate :ci_project_path, to: Gitlab::QA::Runtime::Env # Official dev tag example: -- GitLab From 603a7a08a087dc224fa1ead0e953d9306927e810 Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Wed, 20 Nov 2024 14:35:06 +1300 Subject: [PATCH 07/15] Undoing the removal of new line --- lib/gitlab/qa/component/specs.rb | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/gitlab/qa/component/specs.rb b/lib/gitlab/qa/component/specs.rb index 677968f4..2cc7e842 100644 --- a/lib/gitlab/qa/component/specs.rb +++ b/lib/gitlab/qa/component/specs.rb @@ -149,6 +149,7 @@ module Gitlab def docker_pull_qa_image_if_needed @docker.login(**release.login_params) if release.login_params + @docker.pull(image: qa_image) unless Runtime::Env.skip_pull? end -- GitLab From 330c5b30aa944e7b1b99b90c08a33908b1591710 Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Wed, 20 Nov 2024 17:24:49 +1300 Subject: [PATCH 08/15] Moving position of the constant --- lib/gitlab/qa/release.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index 7b8f2b63..d7d89b6f 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -25,8 +25,6 @@ module Gitlab \z }xi - CUSTOM_OMNIBUS_MIRROR_REGISTRY = %r{\A#{Regexp.escape(COM_REGISTRY)}/.*omnibus-gitlab-mirror\z} - delegate :ci_project_path, to: Gitlab::QA::Runtime::Env # Official dev tag example: @@ -63,6 +61,8 @@ module Gitlab COM_REGISTRY = Gitlab::QA::Runtime::Env.qa_com_registry CUSTOM_REGISTRY = Gitlab::QA::Runtime::Env.qa_custom_registry + CUSTOM_OMNIBUS_MIRROR_REGISTRY = %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} + InvalidImageNameError = Class.new(RuntimeError) attr_reader :release -- GitLab From f66c8db93c7d2befe937c26349f629c1fd636883 Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Fri, 22 Nov 2024 20:03:49 +1300 Subject: [PATCH 09/15] Removed default allocation of custom registry --- lib/gitlab/qa/release.rb | 2 +- lib/gitlab/qa/runtime/env.rb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index d7d89b6f..1591d2d4 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -61,7 +61,7 @@ module Gitlab COM_REGISTRY = Gitlab::QA::Runtime::Env.qa_com_registry CUSTOM_REGISTRY = Gitlab::QA::Runtime::Env.qa_custom_registry - CUSTOM_OMNIBUS_MIRROR_REGISTRY = %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} + CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX = %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} InvalidImageNameError = Class.new(RuntimeError) diff --git a/lib/gitlab/qa/runtime/env.rb b/lib/gitlab/qa/runtime/env.rb index 92755824..30d539ed 100644 --- a/lib/gitlab/qa/runtime/env.rb +++ b/lib/gitlab/qa/runtime/env.rb @@ -441,7 +441,7 @@ module Gitlab end def qa_custom_registry - env_var_value_if_defined('QA_CUSTOM_REGISTRY') || 'registry.gitlab.com' + env_var_value_if_defined('QA_CUSTOM_REGISTRY') end def gitlab_license_mode -- GitLab From 0237e021ff3c629eaab229a6832e1bcadc6d327d Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Mon, 25 Nov 2024 07:59:37 +1300 Subject: [PATCH 10/15] Updated the name of the var --- lib/gitlab/qa/release.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index 1591d2d4..aa2956f6 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -213,7 +213,7 @@ module Gitlab end def omnibus_mirror_fork? - image.start_with?(CUSTOM_OMNIBUS_MIRROR_REGISTRY) + image.start_with?(CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX) end def omnibus_security? -- GitLab From 3238a5e8d3fdfe63bdaaa6dcd5edacde85140e86 Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Mon, 25 Nov 2024 08:09:44 +1300 Subject: [PATCH 11/15] Added a condition to check for nil --- lib/gitlab/qa/release.rb | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index aa2956f6..b8a7935e 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -61,7 +61,11 @@ module Gitlab COM_REGISTRY = Gitlab::QA::Runtime::Env.qa_com_registry CUSTOM_REGISTRY = Gitlab::QA::Runtime::Env.qa_custom_registry - CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX = %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} + CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX ||= begin + if CUSTOM_REGISTRY + %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} + end + end InvalidImageNameError = Class.new(RuntimeError) -- GitLab From a723d3ee158d8633a00cbac51a01944dff4419be Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Mon, 25 Nov 2024 09:55:33 +1300 Subject: [PATCH 12/15] Assinging only if CUSTOM_REGISTRY exists --- lib/gitlab/qa/release.rb | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index b8a7935e..bfd68227 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -61,11 +61,7 @@ module Gitlab COM_REGISTRY = Gitlab::QA::Runtime::Env.qa_com_registry CUSTOM_REGISTRY = Gitlab::QA::Runtime::Env.qa_custom_registry - CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX ||= begin - if CUSTOM_REGISTRY - %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} - end - end + CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX = %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} unless CUSTOM_REGISTRY InvalidImageNameError = Class.new(RuntimeError) -- GitLab From b86018619520287142f156cd72aa81f976346ccf Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Mon, 25 Nov 2024 11:57:41 +1300 Subject: [PATCH 13/15] CHanging the condition --- lib/gitlab/qa/release.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index bfd68227..985b3b7b 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -61,7 +61,7 @@ module Gitlab COM_REGISTRY = Gitlab::QA::Runtime::Env.qa_com_registry CUSTOM_REGISTRY = Gitlab::QA::Runtime::Env.qa_custom_registry - CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX = %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} unless CUSTOM_REGISTRY + CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX = %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} if CUSTOM_REGISTRY InvalidImageNameError = Class.new(RuntimeError) -- GitLab From d71d5f4821b50459e7b69e012b6349d4df37ba19 Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Mon, 25 Nov 2024 12:01:41 +1300 Subject: [PATCH 14/15] Moving regex to method --- lib/gitlab/qa/release.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index 985b3b7b..871686da 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -61,7 +61,7 @@ module Gitlab COM_REGISTRY = Gitlab::QA::Runtime::Env.qa_com_registry CUSTOM_REGISTRY = Gitlab::QA::Runtime::Env.qa_custom_registry - CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX = %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} if CUSTOM_REGISTRY + # CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX = %r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z} if CUSTOM_REGISTRY InvalidImageNameError = Class.new(RuntimeError) @@ -213,7 +213,7 @@ module Gitlab end def omnibus_mirror_fork? - image.start_with?(CUSTOM_OMNIBUS_MIRROR_REGISTRY_REGEX) + image.start_with?(%r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z}) end def omnibus_security? -- GitLab From 3fcffa76f17e584f317a790d891071a7391d226b Mon Sep 17 00:00:00 2001 From: "vishal.s.patel" Date: Mon, 25 Nov 2024 12:06:30 +1300 Subject: [PATCH 15/15] Added a /o option --- lib/gitlab/qa/release.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index 871686da..05489b32 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -213,7 +213,7 @@ module Gitlab end def omnibus_mirror_fork? - image.start_with?(%r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z}) + image.start_with?(%r{\A#{Regexp.escape(CUSTOM_REGISTRY)}/.*omnibus-gitlab-mirror\z}o) end def omnibus_security? -- GitLab