From 8047dd8673d83f26566f3e6326108de2c7d9c878 Mon Sep 17 00:00:00 2001 From: Shinya Maeda Date: Mon, 26 Jun 2023 14:25:59 +0900 Subject: [PATCH] PoC: Support environment keyword in downstream pipelines --- .../projects/environments_controller.rb | 6 +- app/graphql/types/deployment_type.rb | 2 +- app/models/ci/bridge.rb | 16 +- app/models/ci/build.rb | 175 ------------------ app/models/ci/pipeline.rb | 11 +- app/models/ci/processable.rb | 161 +++++++++++++++- app/models/deployment.rb | 6 +- app/serializers/environment_serializer.rb | 2 +- app/services/ci/retry_job_service.rb | 2 +- ...d_service.rb => create_for_job_service.rb} | 44 ++--- .../environments/create_for_build_service.rb | 40 ---- .../environments/create_for_job_service.rb | 40 ++++ app/workers/build_success_worker.rb | 10 +- .../ci/initial_pipeline_process_worker.rb | 2 +- lib/gitlab/ci/config/entry/job.rb | 8 +- lib/gitlab/ci/config/entry/processable.rb | 8 +- .../ci/pipeline/chain/ensure_environments.rb | 4 +- lib/gitlab/ci/variables/builder.rb | 17 ++ 18 files changed, 274 insertions(+), 280 deletions(-) rename app/services/deployments/{create_for_build_service.rb => create_for_job_service.rb} (55%) delete mode 100644 app/services/environments/create_for_build_service.rb create mode 100644 app/services/environments/create_for_job_service.rb diff --git a/app/controllers/projects/environments_controller.rb b/app/controllers/projects/environments_controller.rb index 10d0d03e56dbfe..1aa850a933d7a0 100644 --- a/app/controllers/projects/environments_controller.rb +++ b/app/controllers/projects/environments_controller.rb @@ -117,7 +117,11 @@ def stop action_or_env_url = if stop_actions&.count == 1 - polymorphic_url([project, stop_actions.first]) + if stop_actions.first.instance_of?(::Ci::Build) + polymorphic_url([project, stop_actions.first]) + elsif stop_actions.first.instance_of?(::Ci::Bridge) + project_pipeline_url(project, stop_actions.first.pipeline_id) + end else project_environment_url(project, @environment) end diff --git a/app/graphql/types/deployment_type.rb b/app/graphql/types/deployment_type.rb index 6d895cc81cf1d3..81935741644cb8 100644 --- a/app/graphql/types/deployment_type.rb +++ b/app/graphql/types/deployment_type.rb @@ -55,7 +55,7 @@ class DeploymentType < BaseObject field :job, Types::Ci::JobType, description: 'Pipeline job of the deployment.', - method: :build + method: :deployable field :triggerer, Types::UserType, diff --git a/app/models/ci/bridge.rb b/app/models/ci/bridge.rb index 7cdd0d56a988ff..ecac4871754e4a 100644 --- a/app/models/ci/bridge.rb +++ b/app/models/ci/bridge.rb @@ -71,7 +71,7 @@ def self.with_preloads def self.clone_accessors %i[pipeline project ref tag options name allow_failure stage stage_idx - yaml_variables when description needs_attributes + yaml_variables when environment description needs_attributes scheduling_type ci_stage partition_id].freeze end @@ -180,20 +180,6 @@ def any_unmet_prerequisites? false end - def outdated_deployment? - false - end - - def expanded_environment_name - end - - def persisted_environment - end - - def deployment_job? - false - end - def execute_hooks raise NotImplementedError end diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index bb1bfe8c889298..a7fb54265bb255 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -10,7 +10,6 @@ class Build < Ci::Processable include Presentable include Importable include Ci::HasRef - include Ci::TrackEnvironmentUsage extend ::Gitlab::Utils::Override @@ -32,9 +31,6 @@ class Build < Ci::Processable DEGRADATION_THRESHOLD_VARIABLE_NAME = 'DEGRADATION_THRESHOLD' RUNNERS_STATUS_CACHE_EXPIRATION = 1.minute - DEPLOYMENT_NAMES = %w[deploy release rollout].freeze - - has_one :deployment, as: :deployable, class_name: 'Deployment', inverse_of: :deployable has_one :pending_state, class_name: 'Ci::BuildPendingState', foreign_key: :build_id, inverse_of: :build has_one :queuing_entry, class_name: 'Ci::PendingBuild', foreign_key: :build_id, inverse_of: :build has_one :runtime_metadata, class_name: 'Ci::RunningBuild', foreign_key: :build_id, inverse_of: :build @@ -158,16 +154,9 @@ class Build < Ci::Processable .includes(:metadata, :job_artifacts_metadata) end - scope :with_project_and_metadata, -> do - if Feature.enabled?(:non_public_artifacts, type: :development) - joins(:metadata).includes(:metadata).preload(:project) - end - end - scope :with_artifacts_not_expired, -> { with_downloadable_artifacts.where('artifacts_expire_at IS NULL OR artifacts_expire_at > ?', Time.current) } scope :with_pipeline_locked_artifacts, -> { joins(:pipeline).where('pipeline.locked': Ci::Pipeline.lockeds[:artifacts_locked]) } scope :last_month, -> { where('created_at > ?', Date.today - 1.month) } - scope :manual_actions, -> { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) } scope :scheduled_actions, -> { where(when: :delayed, status: COMPLETED_STATUSES + %i[scheduled]) } scope :ref_protected, -> { where(protected: true) } scope :with_live_trace, -> { where('EXISTS (?)', Ci::BuildTraceChunk.where("#{quoted_table_name}.id = #{Ci::BuildTraceChunk.quoted_table_name}.build_id").select(1)) } @@ -327,7 +316,6 @@ def clone_accessors after_transition any => [:success] do |build| build.run_after_commit do - BuildSuccessWorker.perform_async(id) PagesWorker.perform_async(:deploy, id) if build.pages_generator? end end @@ -345,18 +333,6 @@ def clone_accessors end end end - - # Synchronize Deployment Status - # Please note that the data integirty is not assured because we can't use - # a database transaction due to DB decomposition. - after_transition do |build, transition| - next if transition.loopback? - next unless build.project - - build.run_after_commit do - build.deployment&.sync_status_with(build) - end - end end def self.build_matchers(project) @@ -400,10 +376,6 @@ def detailed_status(current_user) .fabricate! end - def other_manual_actions - pipeline.manual_actions.reject { |action| action.name == name } - end - def other_scheduled_actions pipeline.scheduled_actions.reject { |action| action.name == name } end @@ -428,15 +400,6 @@ def playable? action? && !archived? && (manual? || scheduled? || retryable?) end - def outdated_deployment? - strong_memoize(:outdated_deployment) do - deployment_job? && - incomplete? && - project.ci_forward_deployment_enabled? && - deployment&.older_than_last_successful_deployment? - end - end - def schedulable? self.when == 'delayed' && options[:start_in].present? end @@ -478,94 +441,6 @@ def prerequisites Gitlab::Ci::Build::Prerequisite::Factory.new(self).unmet end - def persisted_environment - return unless has_environment_keyword? - - strong_memoize(:persisted_environment) do - # This code path has caused N+1s in the past, since environments are only indirectly - # associated to builds and pipelines; see https://gitlab.com/gitlab-org/gitlab/-/issues/326445 - # We therefore batch-load them to prevent dormant N+1s until we found a proper solution. - BatchLoader.for(expanded_environment_name).batch(key: project_id) do |names, loader, args| - Environment.where(name: names, project: args[:key]).find_each do |environment| - loader.call(environment.name, environment) - end - end - end - end - - def persisted_environment=(environment) - strong_memoize(:persisted_environment) { environment } - end - - # If build.persisted_environment is a BatchLoader, we need to remove - # the method proxy in order to clone into new item here - # https://github.com/exAspArk/batch-loader/issues/31 - def actual_persisted_environment - persisted_environment.respond_to?(:__sync) ? persisted_environment.__sync : persisted_environment - end - - def expanded_environment_name - return unless has_environment_keyword? - - strong_memoize(:expanded_environment_name) do - # We're using a persisted expanded environment name in order to avoid - # variable expansion per request. - if metadata&.expanded_environment_name.present? - metadata.expanded_environment_name - else - ExpandVariables.expand(environment, -> { simple_variables.sort_and_expand_all }) - end - end - end - - def expanded_kubernetes_namespace - return unless has_environment_keyword? - - namespace = options.dig(:environment, :kubernetes, :namespace) - - if namespace.present? - strong_memoize(:expanded_kubernetes_namespace) do - ExpandVariables.expand(namespace, -> { simple_variables }) - end - end - end - - def has_environment_keyword? - environment.present? - end - - def deployment_job? - has_environment_keyword? && environment_action == 'start' - end - - def stops_environment? - has_environment_keyword? && environment_action == 'stop' - end - - def environment_action - options.fetch(:environment, {}).fetch(:action, 'start') if options - end - - def environment_tier_from_options - options.dig(:environment, :deployment_tier) if options - end - - def environment_tier - environment_tier_from_options || persisted_environment.try(:tier) - end - - def triggered_by?(current_user) - user == current_user - end - - def on_stop - options&.dig(:environment, :on_stop) - end - - def stop_action_successful? - success? - end - ## # All variables, including persisted environment variables. # @@ -577,7 +452,6 @@ def variables .concat(job_jwt_variables) .concat(scoped_variables) .concat(job_variables) - .concat(persisted_environment_variables) end end @@ -601,22 +475,6 @@ def persisted_variables end end - def persisted_environment_variables - Gitlab::Ci::Variables::Collection.new.tap do |variables| - break variables unless persisted? && persisted_environment.present? - - variables.concat(persisted_environment.predefined_variables) - - variables.append(key: 'CI_ENVIRONMENT_ACTION', value: environment_action) - variables.append(key: 'CI_ENVIRONMENT_TIER', value: environment_tier) - - # Here we're passing unexpanded environment_url for runner to expand, - # and we need to make sure that CI_ENVIRONMENT_NAME and - # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded. - variables.append(key: 'CI_ENVIRONMENT_URL', value: environment_url) if environment_url - end - end - def deploy_token_variables Gitlab::Ci::Variables::Collection.new.tap do |variables| break variables unless gitlab_deploy_token @@ -1033,19 +891,6 @@ def report_artifacts job_artifacts.all_reports end - # Virtual deployment status depending on the environment status. - def deployment_status - return unless deployment_job? - - if success? - return successful_deployment_status - elsif failed? - return :failed - end - - :creating - end - # Consider this object to have a structural integrity problems def doom! transaction do @@ -1206,31 +1051,11 @@ def build_data strong_memoize(:build_data) { Gitlab::DataBuilder::Build.build(self) } end - def successful_deployment_status - if deployment&.last? - :last - else - :out_of_date - end - end - def job_artifacts_for_types(report_types) # Use select to leverage cached associations and avoid N+1 queries job_artifacts.select { |artifact| artifact.file_type.in?(report_types) } end - def environment_url - options&.dig(:environment, :url) || persisted_environment&.external_url - end - - def environment_status - strong_memoize(:environment_status) do - if has_environment_keyword? && merge_request - EnvironmentStatus.new(project, persisted_environment, merge_request, pipeline.sha) - end - end - end - def has_expiring_artifacts? artifacts_expire_at.present? && artifacts_expire_at > Time.current end diff --git a/app/models/ci/pipeline.rb b/app/models/ci/pipeline.rb index 69e48cadf975f9..38a3337e0e71f0 100644 --- a/app/models/ci/pipeline.rb +++ b/app/models/ci/pipeline.rb @@ -96,7 +96,7 @@ class Pipeline < Ci::ApplicationRecord has_many :downloadable_artifacts, -> do not_expired.or(where_exists(Ci::Pipeline.artifacts_locked.where("#{Ci::Pipeline.quoted_table_name}.id = #{Ci::Build.quoted_table_name}.commit_id"))).downloadable.with_job end, through: :latest_builds, source: :job_artifacts - has_many :latest_successful_builds, -> { latest.success.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Build' + has_many :latest_successful_jobs, -> { latest.success.with_project_and_metadata }, foreign_key: :commit_id, inverse_of: :pipeline, class_name: 'Ci::Processable' has_many :messages, class_name: 'Ci::PipelineMessage', inverse_of: :pipeline @@ -111,7 +111,7 @@ class Pipeline < Ci::ApplicationRecord has_many :retryable_builds, -> { latest.failed_or_canceled.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :cancelable_statuses, -> { cancelable }, foreign_key: :commit_id, class_name: 'CommitStatus', inverse_of: :pipeline - has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline + has_many :manual_actions, -> { latest.manual_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Processable', inverse_of: :pipeline has_many :scheduled_actions, -> { latest.scheduled_actions.includes(:project) }, foreign_key: :commit_id, class_name: 'Ci::Build', inverse_of: :pipeline has_many :auto_canceled_pipelines, class_name: 'Ci::Pipeline', foreign_key: :auto_canceled_by_id, @@ -967,6 +967,13 @@ def environments_in_self_and_project_descendants(deployment_status: nil) .limit(100) .pluck(:expanded_environment_name) + expanded_environment_names << + bridges_in_self_and_project_descendants.joins(:metadata) + .where.not(Ci::BuildMetadata.table_name => { expanded_environment_name: nil }) + .distinct("#{Ci::BuildMetadata.quoted_table_name}.expanded_environment_name") + .limit(100) + .pluck(:expanded_environment_name) + Environment.where(project: project, name: expanded_environment_names).with_deployment(sha, status: deployment_status) end diff --git a/app/models/ci/processable.rb b/app/models/ci/processable.rb index 4c421f066f97f9..e74e0b3e07e3a5 100644 --- a/app/models/ci/processable.rb +++ b/app/models/ci/processable.rb @@ -7,7 +7,11 @@ class Processable < ::CommitStatus include Gitlab::Utils::StrongMemoize include FromUnion extend ::Gitlab::Utils::Override + include Ci::TrackEnvironmentUsage + DEPLOYMENT_NAMES = %w[deploy release rollout].freeze + + has_one :deployment, as: :deployable, class_name: 'Deployment', inverse_of: :deployable has_one :resource, class_name: 'Ci::Resource', foreign_key: 'build_id', inverse_of: :processable has_one :sourced_pipeline, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_job_id, inverse_of: :source_job @@ -16,6 +20,7 @@ class Processable < ::CommitStatus accepts_nested_attributes_for :needs scope :preload_needs, -> { preload(:needs) } + scope :manual_actions, -> { where(when: :manual, status: COMPLETED_STATUSES + %i[manual]) } scope :with_needs, -> (names = nil) do needs = Ci::BuildNeed.scoped_build.select(1) @@ -29,6 +34,12 @@ class Processable < ::CommitStatus where('NOT EXISTS (?)', needs) end + scope :with_project_and_metadata, -> do + if Feature.enabled?(:non_public_artifacts, type: :development) + joins(:metadata).includes(:metadata).preload(:project) + end + end + state_machine :status do event :enqueue do transition [:created, :skipped, :manual, :scheduled] => :waiting_for_resource, if: :with_resource_group? @@ -70,6 +81,24 @@ class Processable < ::CommitStatus .perform_async(processable.resource_group_id) end end + + after_transition any => [:success] do |processable| + processable.run_after_commit do + BuildSuccessWorker.perform_async(id) + end + end + + # Synchronize Deployment Status + # Please note that the data integirty is not assured because we can't use + # a database transaction due to DB decomposition. + after_transition do |processable, transition| + next if transition.loopback? + next unless processable.project + + processable.run_after_commit do + processable.deployment&.sync_status_with(processable) + end + end end def self.select_with_aggregated_needs(project) @@ -142,12 +171,84 @@ def when read_attribute(:when) || 'on_success' end + def persisted_environment + return unless has_environment_keyword? + + strong_memoize(:persisted_environment) do + # This code path has caused N+1s in the past, since environments are only indirectly + # associated to builds and pipelines; see https://gitlab.com/gitlab-org/gitlab/-/issues/326445 + # We therefore batch-load them to prevent dormant N+1s until we found a proper solution. + BatchLoader.for(expanded_environment_name).batch(key: project_id) do |names, loader, args| + Environment.where(name: names, project: args[:key]).find_each do |environment| + loader.call(environment.name, environment) + end + end + end + end + + def persisted_environment=(environment) + strong_memoize(:persisted_environment) { environment } + end + + # If build.persisted_environment is a BatchLoader, we need to remove + # the method proxy in order to clone into new item here + # https://github.com/exAspArk/batch-loader/issues/31 + def actual_persisted_environment + persisted_environment.respond_to?(:__sync) ? persisted_environment.__sync : persisted_environment + end + def expanded_environment_name - raise NotImplementedError + return unless has_environment_keyword? + + strong_memoize(:expanded_environment_name) do + # We're using a persisted expanded environment name in order to avoid + # variable expansion per request. + if metadata&.expanded_environment_name.present? + metadata.expanded_environment_name + else + ExpandVariables.expand(environment, -> { simple_variables.sort_and_expand_all }) + end + end end - def persisted_environment - raise NotImplementedError + def expanded_kubernetes_namespace + return unless has_environment_keyword? + + namespace = options.dig(:environment, :kubernetes, :namespace) + + if namespace.present? + strong_memoize(:expanded_kubernetes_namespace) do + ExpandVariables.expand(namespace, -> { simple_variables }) + end + end + end + + def environment_action + options.fetch(:environment, {}).fetch(:action, 'start') if options + end + + def environment_tier_from_options + options.dig(:environment, :deployment_tier) if options + end + + def environment_tier + environment_tier_from_options || persisted_environment.try(:tier) + end + + def triggered_by?(current_user) + user == current_user + end + + def on_stop + options&.dig(:environment, :on_stop) + end + + def stop_action_successful? + success? + end + + def other_manual_actions + pipeline.manual_actions.reject { |action| action.name == name } end override :all_met_to_become_pending? @@ -159,6 +260,60 @@ def with_resource_group? self.resource_group_id.present? end + def has_environment_keyword? + environment.present? + end + + def deployment_job? + has_environment_keyword? && environment_action == 'start' + end + + def stops_environment? + has_environment_keyword? && environment_action == 'stop' + end + + def environment_url + options&.dig(:environment, :url) || persisted_environment&.external_url + end + + def environment_status + strong_memoize(:environment_status) do + if has_environment_keyword? && merge_request + EnvironmentStatus.new(project, persisted_environment, merge_request, pipeline.sha) + end + end + end + + def outdated_deployment? + strong_memoize(:outdated_deployment) do + deployment_job? && + incomplete? && + project.ci_forward_deployment_enabled? && + deployment&.older_than_last_successful_deployment? + end + end + + # Virtual deployment status depending on the environment status. + def deployment_status + return unless deployment_job? + + if success? + return successful_deployment_status + elsif failed? + return :failed + end + + :creating + end + + def successful_deployment_status + if deployment&.last? + :last + else + :out_of_date + end + end + # Overriding scheduling_type enum's method for nil `scheduling_type`s def scheduling_type_dag? scheduling_type.nil? ? find_legacy_scheduling_type == :dag : super diff --git a/app/models/deployment.rb b/app/models/deployment.rb index 5af0216a3ca850..3908a4c0136924 100644 --- a/app/models/deployment.rb +++ b/app/models/deployment.rb @@ -184,9 +184,9 @@ def self.last_for_environment(environment) # - deploy job B => production environment # In this case, `last_deployment_group` returns both deployments. # - # NOTE: Preload environment.last_deployment and pipeline.latest_successful_builds prior to avoid N+1. + # NOTE: Preload environment.last_deployment and pipeline.latest_successful_jobs prior to avoid N+1. def self.last_deployment_group_for_environment(env) - return self.none unless env.last_deployment_pipeline&.latest_successful_builds&.present? + return self.none unless env.last_deployment_pipeline&.latest_successful_jobs&.present? BatchLoader.for(env).batch(default_value: self.none) do |environments, loader| latest_successful_build_ids = [] @@ -196,7 +196,7 @@ def self.last_deployment_group_for_environment(env) environments_hash[environment.id] = environment # Refer comment note above, if not preloaded this can lead to N+1. - latest_successful_build_ids << environment.last_deployment_pipeline.latest_successful_builds.map(&:id) + latest_successful_build_ids << environment.last_deployment_pipeline.latest_successful_jobs.map(&:id) end Deployment diff --git a/app/serializers/environment_serializer.rb b/app/serializers/environment_serializer.rb index d7820dff6ef1b1..8f3aeea2eedc55 100644 --- a/app/serializers/environment_serializer.rb +++ b/app/serializers/environment_serializer.rb @@ -94,7 +94,7 @@ def deployment_associations pipeline: { manual_actions: [:metadata, :deployment], scheduled_actions: [:metadata], - latest_successful_builds: [] + latest_successful_jobs: [] }, project: project_associations } diff --git a/app/services/ci/retry_job_service.rb b/app/services/ci/retry_job_service.rb index e3cbba6de23aa0..14ea09f17a05fa 100644 --- a/app/services/ci/retry_job_service.rb +++ b/app/services/ci/retry_job_service.rb @@ -39,7 +39,7 @@ def clone!(job, variables: [], enqueue_if_actionable: false, start_pipeline: fal ::Ci::CopyCrossDatabaseAssociationsService.new.execute(job, new_job) - ::Deployments::CreateForBuildService.new.execute(new_job) + ::Deployments::CreateForJobService.new.execute(new_job) ::MergeRequests::AddTodoWhenBuildFailsService .new(project: project) diff --git a/app/services/deployments/create_for_build_service.rb b/app/services/deployments/create_for_job_service.rb similarity index 55% rename from app/services/deployments/create_for_build_service.rb rename to app/services/deployments/create_for_job_service.rb index b58aa50a66f897..be1634fae1c7f9 100644 --- a/app/services/deployments/create_for_build_service.rb +++ b/app/services/deployments/create_for_job_service.rb @@ -1,36 +1,36 @@ # frozen_string_literal: true module Deployments - # This class creates a deployment record for a build (a pipeline job). - class CreateForBuildService + # This class creates a deployment record for a job (a pipeline job). + class CreateForJobService DeploymentCreationError = Class.new(StandardError) - def execute(build) - return unless build.instance_of?(::Ci::Build) && build.persisted_environment.present? + def execute(job) + return unless job.kind_of?(::Ci::Processable) && job.persisted_environment.present? - environment = build.actual_persisted_environment + environment = job.actual_persisted_environment - deployment = to_resource(build, environment) + deployment = to_resource(job, environment) return unless deployment deployment.save! - build.association(:deployment).target = deployment - build.association(:deployment).loaded! + job.association(:deployment).target = deployment + job.association(:deployment).loaded! deployment rescue ActiveRecord::RecordInvalid => e Gitlab::ErrorTracking.track_and_raise_for_dev_exception( - DeploymentCreationError.new(e.message), build_id: build.id) + DeploymentCreationError.new(e.message), job_id: job.id) end private - def to_resource(build, environment) - return build.deployment if build.deployment - return unless build.deployment_job? + def to_resource(job, environment) + return job.deployment if job.deployment + return unless job.deployment_job? - deployment = ::Deployment.new(attributes(build, environment)) + deployment = ::Deployment.new(attributes(job, environment)) # If there is a validation error on environment creation, such as # the name contains invalid character, the job will fall back to a @@ -42,7 +42,7 @@ def to_resource(build, environment) deployment.cluster_id = cluster.id deployment.deployment_cluster = ::DeploymentCluster.new( cluster_id: cluster.id, - kubernetes_namespace: cluster.kubernetes_namespace_for(deployment.environment, deployable: build) + kubernetes_namespace: cluster.kubernetes_namespace_for(deployment.environment, deployable: job) ) end @@ -53,16 +53,16 @@ def to_resource(build, environment) deployment end - def attributes(build, environment) + def attributes(job, environment) { - project: build.project, + project: job.project, environment: environment, - deployable: build, - user: build.user, - ref: build.ref, - tag: build.tag, - sha: build.sha, - on_stop: build.on_stop + deployable: job, + user: job.user, + ref: job.ref, + tag: job.tag, + sha: job.sha, + on_stop: job.on_stop } end end diff --git a/app/services/environments/create_for_build_service.rb b/app/services/environments/create_for_build_service.rb deleted file mode 100644 index ff4da212002b03..00000000000000 --- a/app/services/environments/create_for_build_service.rb +++ /dev/null @@ -1,40 +0,0 @@ -# frozen_string_literal: true - -module Environments - # This class creates an environment record for a build (a pipeline job). - class CreateForBuildService - def execute(build) - return unless build.instance_of?(::Ci::Build) && build.has_environment_keyword? - - environment = to_resource(build) - - if environment.persisted? - build.persisted_environment = environment - build.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name }) - else - build.assign_attributes(status: :failed, failure_reason: :environment_creation_failure) - end - - environment - end - - private - - # rubocop: disable Performance/ActiveRecordSubtransactionMethods - def to_resource(build) - build.project.environments.safe_find_or_create_by(name: build.expanded_environment_name) do |environment| - # Initialize the attributes at creation - environment.auto_stop_in = expanded_auto_stop_in(build) - environment.tier = build.environment_tier_from_options - environment.merge_request = build.pipeline.merge_request - end - end - # rubocop: enable Performance/ActiveRecordSubtransactionMethods - - def expanded_auto_stop_in(build) - return unless build.environment_auto_stop_in - - ExpandVariables.expand(build.environment_auto_stop_in, -> { build.simple_variables.sort_and_expand_all }) - end - end -end diff --git a/app/services/environments/create_for_job_service.rb b/app/services/environments/create_for_job_service.rb new file mode 100644 index 00000000000000..e6c85f66d80cad --- /dev/null +++ b/app/services/environments/create_for_job_service.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module Environments + # This class creates an environment record for a job (a pipeline job). + class CreateForJobService + def execute(job) + return unless job.kind_of?(::Ci::Processable) && job.has_environment_keyword? + + environment = to_resource(job) + + if environment.persisted? + job.persisted_environment = environment + job.assign_attributes(metadata_attributes: { expanded_environment_name: environment.name }) + else + job.assign_attributes(status: :failed, failure_reason: :environment_creation_failure) + end + + environment + end + + private + + # rubocop: disable Performance/ActiveRecordSubtransactionMethods + def to_resource(job) + job.project.environments.safe_find_or_create_by(name: job.expanded_environment_name) do |environment| + # Initialize the attributes at creation + environment.auto_stop_in = expanded_auto_stop_in(job) + environment.tier = job.environment_tier_from_options + environment.merge_request = job.pipeline.merge_request + end + end + # rubocop: enable Performance/ActiveRecordSubtransactionMethods + + def expanded_auto_stop_in(job) + return unless job.environment_auto_stop_in + + ExpandVariables.expand(job.environment_auto_stop_in, -> { job.simple_variables.sort_and_expand_all }) + end + end +end diff --git a/app/workers/build_success_worker.rb b/app/workers/build_success_worker.rb index 247105d2a1a255..77fc9ce6dc0542 100644 --- a/app/workers/build_success_worker.rb +++ b/app/workers/build_success_worker.rb @@ -11,15 +11,15 @@ class BuildSuccessWorker # rubocop:disable Scalability/IdempotentWorker queue_namespace :pipeline_processing urgency :high - def perform(build_id) - Ci::Build.find_by_id(build_id).try do |build| - stop_environment(build) if build.stops_environment? && build.stop_action_successful? + def perform(job_id) + Ci::Processable.find_by_id(job_id).try do |job| + stop_environment(job) if job.stops_environment? && job.stop_action_successful? end end private - def stop_environment(build) - build.persisted_environment.fire_state_event(:stop_complete) + def stop_environment(job) + job.persisted_environment.fire_state_event(:stop_complete) end end diff --git a/app/workers/ci/initial_pipeline_process_worker.rb b/app/workers/ci/initial_pipeline_process_worker.rb index 52a4f075cf06fb..067dbb7492fc95 100644 --- a/app/workers/ci/initial_pipeline_process_worker.rb +++ b/app/workers/ci/initial_pipeline_process_worker.rb @@ -32,7 +32,7 @@ def create_deployments!(pipeline) end def create_deployment(build) - ::Deployments::CreateForBuildService.new.execute(build) + ::Deployments::CreateForJobService.new.execute(build) end end end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index d31d1b366c3fe2..bc0c8016934e1a 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -13,7 +13,7 @@ class Job < ::Gitlab::Config::Entry::Node ALLOWED_WHEN = %w[on_success on_failure always manual delayed].freeze ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script hooks - environment coverage retry parallel interruptible timeout + coverage retry parallel interruptible timeout release id_tokens publish].freeze validations do @@ -102,10 +102,6 @@ class Job < ::Gitlab::Config::Entry::Node metadata: { allowed_needs: %i[job cross_dependency] }, inherit: false - entry :environment, Entry::Environment, - description: 'Environment configuration for this job.', - inherit: false - entry :coverage, Entry::Coverage, description: 'Coverage configuration for this job.', inherit: false @@ -160,8 +156,6 @@ def value when: self.when, start_in: self.start_in, dependencies: dependencies, - environment: environment_defined? ? environment_value : nil, - environment_name: environment_defined? ? environment_value[:name] : nil, coverage: coverage_defined? ? coverage_value : nil, retry: retry_defined? ? retry_value : nil, parallel: has_parallel? ? parallel_value : nil, diff --git a/lib/gitlab/ci/config/entry/processable.rb b/lib/gitlab/ci/config/entry/processable.rb index e0f0903174cede..88734ac11862fe 100644 --- a/lib/gitlab/ci/config/entry/processable.rb +++ b/lib/gitlab/ci/config/entry/processable.rb @@ -15,7 +15,7 @@ module Processable include ::Gitlab::Config::Entry::Inheritable PROCESSABLE_ALLOWED_KEYS = %i[extends stage only except rules variables - inherit allow_failure when needs resource_group].freeze + inherit allow_failure when needs resource_group environment].freeze MAX_NESTING_LEVEL = 10 included do @@ -68,6 +68,10 @@ module Processable inherit: false, default: {} + entry :environment, Entry::Environment, + description: 'Environment configuration for this job.', + inherit: false + attributes :extends, :rules, :resource_group end @@ -125,6 +129,8 @@ def value root_variables_inheritance: root_variables_inheritance, only: only_value, except: except_value, + environment: environment_defined? ? environment_value : nil, + environment_name: environment_defined? ? environment_value[:name] : nil, resource_group: resource_group }.compact end diff --git a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb index ebea6a538efa11..1269257f44d244 100644 --- a/lib/gitlab/ci/pipeline/chain/ensure_environments.rb +++ b/lib/gitlab/ci/pipeline/chain/ensure_environments.rb @@ -15,8 +15,8 @@ def break? private - def ensure_environment(build) - ::Environments::CreateForBuildService.new.execute(build) + def ensure_environment(job) + ::Environments::CreateForJobService.new.execute(job) end end end diff --git a/lib/gitlab/ci/variables/builder.rb b/lib/gitlab/ci/variables/builder.rb index cae3a966bc61ee..5c37e36ef9115b 100644 --- a/lib/gitlab/ci/variables/builder.rb +++ b/lib/gitlab/ci/variables/builder.rb @@ -31,6 +31,7 @@ def scoped_variables(job, environment:, dependencies:) variables.concat(pipeline.variables) variables.concat(pipeline_schedule_variables) variables.concat(release_variables) + variables.concat(persisted_environment_variables(job)) if environment.present? end end @@ -64,6 +65,22 @@ def kubernetes_variables(environment:, job:) end end + def persisted_environment_variables(job) + Gitlab::Ci::Variables::Collection.new.tap do |variables| + break variables unless job.persisted_environment.present? + + variables.concat(job.persisted_environment.predefined_variables) + + variables.append(key: 'CI_ENVIRONMENT_ACTION', value: job.environment_action) + variables.append(key: 'CI_ENVIRONMENT_TIER', value: job.environment_tier) + + # Here we're passing unexpanded environment_url for runner to expand, + # and we need to make sure that CI_ENVIRONMENT_NAME and + # CI_ENVIRONMENT_SLUG so on are available for the URL be expanded. + variables.append(key: 'CI_ENVIRONMENT_URL', value: job.environment_url) if job.environment_url + end + end + def deployment_variables(environment:, job:) return [] unless environment -- GitLab