diff --git a/lib/gitlab/qa/component/base.rb b/lib/gitlab/qa/component/base.rb index 7d66bf36816d6bf0ed974b39d094be43e781259f..988e8782f5a00c86098b9daaa8efabbc743c6aed 100644 --- a/lib/gitlab/qa/component/base.rb +++ b/lib/gitlab/qa/component/base.rb @@ -208,10 +208,11 @@ module Gitlab # for scenarios where a service fails during startup, attempt to retry to avoid flaky failures if (retries += 1) < 3 unless reconfigure_log_file.nil? - Runtime::Logger.info( - "Follow the document " \ - "https://gitlab.com/gitlab-org/quality/runbooks/-/blob/main/debug_orchestrated_test_locally/ " \ - "for debugging the test failure locally.") + Runtime::Logger.info(%( + Follow the document + https://gitlab.com/gitlab-org/quality/runbooks/-/blob/88787f79d4af50d54bb1e0249fe11227288067d3/debug_orchestrated_test_locally/running_update-major_and_update-minor_locally.md + for debugging the test failure locally. + )) # Tailing the reconfigure logs after retries are over and before raising exception Runtime::Logger.info("Tail of the reconfigure log file, see artifacts for full log: #{reconfigure_log_file}") diff --git a/lib/gitlab/qa/component/gitaly.rb b/lib/gitlab/qa/component/gitaly.rb index 53edc009d995a59e8362f42bb06da70a98fbbed8..533ae87bd7c2ee23d1d4463bbbfdb6a889e0a62b 100644 --- a/lib/gitlab/qa/component/gitaly.rb +++ b/lib/gitlab/qa/component/gitaly.rb @@ -5,6 +5,7 @@ module Gitlab module Component class Gitaly < Base extend Forwardable + using Rainbow attr_reader :release attr_accessor :gitaly_port, :gitlab_name, :cluster_config diff --git a/lib/gitlab/qa/component/gitaly_cluster.rb b/lib/gitlab/qa/component/gitaly_cluster.rb index 8c9ff68e46b260f80198f70653be70ca6adcaa6a..73add5f98c2341106fffd301c5c2bf12a0b74300 100644 --- a/lib/gitlab/qa/component/gitaly_cluster.rb +++ b/lib/gitlab/qa/component/gitaly_cluster.rb @@ -43,11 +43,28 @@ module Gitlab end include Scenario::Actable + using Rainbow attr_accessor :release, :exec_commands, :gitlab_name, :config attr_reader :gitaly_primary_node, :gitaly_secondary_node, :gitaly_tertiary_node, :praefect_node, :database_node + # Helper configuration for omnibus config to disable all non GitalyCluster related omnibus services + def self.disable_other_omnibus_services + <<~OMNIBUS + postgresql['enable'] = false; + redis['enable'] = false; + nginx['enable'] = false; + puma['enable'] = false; + sidekiq['enable'] = false; + gitlab_workhorse['enable'] = false; + gitlab_rails['rake_cache_clear'] = false; + gitlab_rails['auto_migrate'] = false; + gitlab_exporter['enable'] = false; + gitlab_kas['enable'] = false; + OMNIBUS + end + def initialize(config = GitalyClusterConfig.new) @spec_suite = 'Test::Instance::All' @env = {} @@ -134,22 +151,6 @@ module Gitlab praefect.instance(skip_teardown: true) end end - - # Helper configuration for omnibus config to disable all non GitalyCluster related omnibus services - def self.disable_other_omnibus_services - <<~OMNIBUS - postgresql['enable'] = false; - redis['enable'] = false; - nginx['enable'] = false; - puma['enable'] = false; - sidekiq['enable'] = false; - gitlab_workhorse['enable'] = false; - gitlab_rails['rake_cache_clear'] = false; - gitlab_rails['auto_migrate'] = false; - gitlab_exporter['enable'] = false; - gitlab_kas['enable'] = false; - OMNIBUS - end end end end diff --git a/lib/gitlab/qa/component/gitlab.rb b/lib/gitlab/qa/component/gitlab.rb index 62e592b6ae0dae460a3fcb1c0cbf52422f9ab727..f6aec39d958a2207b44f53be97c094d0c4776509 100644 --- a/lib/gitlab/qa/component/gitlab.rb +++ b/lib/gitlab/qa/component/gitlab.rb @@ -15,6 +15,7 @@ module Gitlab module Component class Gitlab < Base extend Forwardable + using Rainbow attr_reader :release, @@ -195,31 +196,11 @@ module Gitlab ) end - # Path of the log file to write to - # @note - # if an error occurs during #reconfigure, - # "retry-{n}" is prefixed to the file name where {n} is retry number - # @return [String] the path to the log file - def log_file - retries = 0 - log_file = "#{Runtime::Env.host_artifacts_dir}/#{name}-reconfigure.log" - while File.exist?(log_file) - break unless File.exist?(log_file) - - retries += 1 - log_file = "#{Runtime::Env.host_artifacts_dir}/#{name}-retry-#{retries}-reconfigure.log" - end - - log_file - end - def get_reconfigure_log_file_from_artifact all_reconfigure_log_file = Dir["#{Runtime::Env.host_artifacts_dir}/*reconfigure.log"].sort_by { |f| File.mtime(f) } all_reconfigure_log_file.last end - private :log_file - def reconfigure setup_omnibus log_file_path = log_file @@ -311,6 +292,26 @@ module Gitlab super end + # Path of the log file to write to + # @note + # if an error occurs during #reconfigure, + # "retry-{n}" is prefixed to the file name where {n} is retry number + # @return [String] the path to the log file + def log_file + retries = 0 + log_file = "#{Runtime::Env.host_artifacts_dir}/#{name}-reconfigure.log" + while File.exist?(log_file) + break unless File.exist?(log_file) + + retries += 1 + log_file = "#{Runtime::Env.host_artifacts_dir}/#{name}-retry-#{retries}-reconfigure.log" + end + + log_file + end + + private :log_file + private attr_reader :cert_volumes diff --git a/lib/gitlab/qa/component/praefect.rb b/lib/gitlab/qa/component/praefect.rb index 3e9850e005888534cea1760727fe52698ddba5fd..ce338b1dcec2255f77239ef3c5e2e58f70a6cceb 100644 --- a/lib/gitlab/qa/component/praefect.rb +++ b/lib/gitlab/qa/component/praefect.rb @@ -5,6 +5,7 @@ module Gitlab module Component class Praefect < Base extend Forwardable + using Rainbow attr_reader :release attr_accessor :cluster_config diff --git a/lib/gitlab/qa/component/runner_ops.rb b/lib/gitlab/qa/component/runner_ops.rb index 071c5a4852c9a01ba99ebbba284fcf4defd8357c..ae1bc09de294d792ddfd09cca7639c81ca97a2a7 100644 --- a/lib/gitlab/qa/component/runner_ops.rb +++ b/lib/gitlab/qa/component/runner_ops.rb @@ -86,7 +86,7 @@ module Gitlab ) <<~CMD.strip - printf '#{runner_config.chomp.gsub(/\n/, '\\n').gsub('"', '\"')}' > /etc/gitlab-runner/config.toml && + printf '#{runner_config.chomp.gsub("\n", '\\n').gsub('"', '\"')}' > /etc/gitlab-runner/config.toml && gitlab-runner register #{registration_args} && gitlab-runner run CMD diff --git a/lib/gitlab/qa/docker/command.rb b/lib/gitlab/qa/docker/command.rb index c8df36dadff089c53d094bbbc764165484f3d552..228e8717b34392cc82a49bbb99dd4aec6935904b 100644 --- a/lib/gitlab/qa/docker/command.rb +++ b/lib/gitlab/qa/docker/command.rb @@ -6,6 +6,10 @@ module Gitlab class Command attr_reader :args + def self.execute(cmd, mask_secrets: nil, &block) + new(cmd, mask_secrets: mask_secrets).execute!(&block) + end + # Shell command # # @param [] cmd @@ -52,10 +56,6 @@ module Gitlab raise e end - - def self.execute(cmd, mask_secrets: nil, &block) - new(cmd, mask_secrets: mask_secrets).execute!(&block) - end end end end diff --git a/lib/gitlab/qa/release.rb b/lib/gitlab/qa/release.rb index 191f053529288b1df63c336313a6064be629109a..50d5cace96713df162d5e291930d536ecc2573d3 100644 --- a/lib/gitlab/qa/release.rb +++ b/lib/gitlab/qa/release.rb @@ -146,14 +146,17 @@ module Gitlab end def qa_tag + match_dev_official_tag_regexp = tag.match(DEV_OFFICIAL_TAG_REGEX) + match_dev_regex = tag.match(DEV_TAG_REGEX) + # Case 1: Official packages (e.g., 11.1.0-rc12.ee.0 or 12.5.4-ce.1) - if match_data = tag.match(DEV_OFFICIAL_TAG_REGEX) - version = match_data[:version] - edition = match_data[:edition] + if match_dev_official_tag_regexp + version = match_dev_official_tag_regexp[:version] + edition = match_dev_official_tag_regexp[:edition] "#{version}-#{edition}" # Case 2: Auto-deploy packages from dev.gitlab.org (e.g., 12.1.201906121026-325a6632895.b340d0bd35d) - elsif match_data = tag.match(DEV_TAG_REGEX) - match_data[:gitlab_ref] + elsif match_dev_regex + match_dev_regex[:gitlab_ref] else tag.sub(/[-.]([ce]e)(\.(\d+))?\z/, '-\1') end diff --git a/lib/gitlab/qa/runner.rb b/lib/gitlab/qa/runner.rb index 27245f0330bd084f840c7069cddb06d9597332ca..30d0a397ba34ceb620ba5fa3afb16c16d502d719 100644 --- a/lib/gitlab/qa/runner.rb +++ b/lib/gitlab/qa/runner.rb @@ -47,7 +47,7 @@ module Gitlab '--qa-image QA_IMAGE', String, "Specifies a QA image to be used instead of inferring it from the GitLab image." \ - "See Gitlab::QA::Release#qa_image" + "See Gitlab::QA::Release#qa_image" ) do |value| Runtime::Scenario.define(:qa_image, value) end diff --git a/lib/gitlab/qa/runtime/env.rb b/lib/gitlab/qa/runtime/env.rb index 918879e4923193fa2afd953647bb6ab7383a4b2a..449b17a300b77c32eb3218db99940d08aa4e1dfa 100644 --- a/lib/gitlab/qa/runtime/env.rb +++ b/lib/gitlab/qa/runtime/env.rb @@ -497,12 +497,12 @@ module Gitlab end def env_var_value_if_defined(variable) - return ENV.fetch(variable, nil) if env_var_value_valid?(variable) + ENV.fetch(variable, nil) if env_var_value_valid?(variable) end def env_var_name_if_defined(variable) # Pass through the variables if they are defined and not empty in the environment - return "$#{variable}" if env_var_value_valid?(variable) + "$#{variable}" if env_var_value_valid?(variable) end def method_name_and_value(env_var_name, attributes, name_as_value: false) diff --git a/lib/gitlab/qa/runtime/omnibus_configuration.rb b/lib/gitlab/qa/runtime/omnibus_configuration.rb index 8a453d65596cb1e88588f998e02d096899ebbe3c..04727fb5f3f76f7d289094f05050563cf09db8d7 100644 --- a/lib/gitlab/qa/runtime/omnibus_configuration.rb +++ b/lib/gitlab/qa/runtime/omnibus_configuration.rb @@ -9,6 +9,8 @@ module Gitlab module QA module Runtime class OmnibusConfiguration + ERB_PATTERN = /<%=?\s?(\S+)\s?%>/ + # @param prefixed_config The configuration to be prefixed to the new configuration def initialize(prefixed_config = nil) @config = ["# Generated by GitLab QA Omnibus Configurator at #{Time.now.strftime('%Y-%m-%d %H:%M:%S')}"] @@ -27,8 +29,6 @@ module Gitlab sanitize!.join("\n") end - ERB_PATTERN = /<%=?\s?(\S+)\s?%>/ - # Execute the ERB code to produce completed template # @example # external_url '<%= gitlab.address %>' #=> external_url 'http://gitlab-ee-09d62235.test' diff --git a/lib/gitlab/qa/runtime/omnibus_configurations/default.rb b/lib/gitlab/qa/runtime/omnibus_configurations/default.rb index af13dd00d31bc9fee3f972318f1d9cd817dd207c..f3ab6ea827dbba416e8531790b771878574b48ba 100644 --- a/lib/gitlab/qa/runtime/omnibus_configurations/default.rb +++ b/lib/gitlab/qa/runtime/omnibus_configurations/default.rb @@ -7,6 +7,10 @@ module Gitlab # Default Configuration for Omnibus # All runs will include this configuration class Default < Runtime::OmnibusConfiguration + def self.configuration + new.configuration + end + def configuration <<~OMNIBUS gitlab_rails['gitlab_default_theme'] = 10 # Light Red Theme @@ -15,10 +19,6 @@ module Gitlab gitlab_rails['initial_root_password'] = '#{Runtime::Env.admin_password}' # Initial root password OMNIBUS end - - def self.configuration - new.configuration - end end end end diff --git a/lib/gitlab/qa/support/gitlab_version_info.rb b/lib/gitlab/qa/support/gitlab_version_info.rb index 04f37a70368149efb97e71e421b887b840a8c768..e3aa1ed17dfcae1a88ee47c2b1196652fac5075a 100644 --- a/lib/gitlab/qa/support/gitlab_version_info.rb +++ b/lib/gitlab/qa/support/gitlab_version_info.rb @@ -6,8 +6,10 @@ module Gitlab class GitlabVersionInfo VERSION_PATTERN = /^(?\d+\.\d+\.\d+)/ COMPONENT_PATTERN = /^(?\d+)\.(?\d+)\.(?\d+)/ - VersionNotFoundError = Class.new(RuntimeError) + MAX_TAGS_HTTP_REQUESTS = 50 + # https://docs.docker.com/docker-hub/api/latest/#tag/images/operation/GetNamespacesRepositoriesImages + TAGS_PER_PAGE = 100 # Get previous gitlab version # @@ -88,10 +90,6 @@ module Gitlab private - MAX_TAGS_HTTP_REQUESTS = 50 - # https://docs.docker.com/docker-hub/api/latest/#tag/images/operation/GetNamespacesRepositoriesImages - TAGS_PER_PAGE = 100 - attr_reader :current_version, :edition, :logger # Current versions major version diff --git a/lib/gitlab/qa/support/invalid_response_error.rb b/lib/gitlab/qa/support/invalid_response_error.rb index 6ff6102ae3f4cb46b87941588112f7cc4c86f5a7..8e272e064379c99010544867e3fdd69c8a8f19f8 100644 --- a/lib/gitlab/qa/support/invalid_response_error.rb +++ b/lib/gitlab/qa/support/invalid_response_error.rb @@ -9,7 +9,7 @@ module Gitlab def initialize(address, response) @response = response - super "Invalid response received from #{address}" + super("Invalid response received from #{address}") end end end diff --git a/rubocop/cop/gitlab/dangerous_interpolation.rb b/rubocop/cop/gitlab/dangerous_interpolation.rb index 772ad04eada45a3a9c4afb7447eb5b3a2ab5f5db..ef13d5e5498e598064de5170a4109160c61d712b 100644 --- a/rubocop/cop/gitlab/dangerous_interpolation.rb +++ b/rubocop/cop/gitlab/dangerous_interpolation.rb @@ -17,7 +17,8 @@ module RuboCop class DangerousInterpolation < Base MSG = 'Sensitive variables should not be logged. ' \ - 'Please ensure that the interpolated variable is not sensitive. If not, add it to `ENV_VARIABLES` in the cop class.' + 'Please ensure that the interpolated variable is not sensitive.' \ + 'If not, add it to `ENV_VARIABLES` in the cop class.' # These values are taken from /lib/gitlab/qa/runtime/env.rb. Sensitive variables have been removed. ENV_VARIABLES = {