From 9fdaf3b116af4bf9ad9315da20353c2144f92b70 Mon Sep 17 00:00:00 2001 From: Kassio Borges Date: Wed, 16 Aug 2023 21:45:26 +0100 Subject: [PATCH 1/6] Add `pages_path_prefix` job option As part of the introduction of the [GitLab Pages Multiple Versions][] Support, we need to introduce `pages_path_prefix` as a pages job option on gitlab CI. This is a iteration step towards the [GitLab Pages Multiple Versions][]. The main goal here is to save the `pages_path_prefix` on `PagesDeployment` records. The `PagesDeployment#path_prefix` was added on https://gitlab.com/gitlab-org/gitlab/-/merge_requests/128930. This value will be used on https://gitlab.com/gitlab-org/gitlab/-/issues/416494. Related to: https://gitlab.com/gitlab-org/gitlab/-/issues/416492 [GitLab Pages Multiple Versions]: https://gitlab.com/groups/gitlab-org/-/epics/10914 Changelog: added --- app/assets/javascripts/editor/schema/ci.json | 4 + app/models/ci/build.rb | 5 +- app/services/projects/update_pages_service.rb | 23 ++++-- ee/app/models/ee/ci/build.rb | 11 +++ .../ee/projects/update_pages_service.rb | 16 ++++ ee/lib/ee/gitlab/ci/config/entry/job.rb | 30 +++++-- ee/lib/ee/gitlab/ci/yaml_processor/result.rb | 5 +- .../ci/config/entry/pages_path_prefix.rb | 20 +++++ ee/spec/models/ci/build_spec.rb | 32 ++++++++ .../ee/projects/update_pages_service_spec.rb | 73 +++++++++++++++++ lib/gitlab/ci/config/entry/job.rb | 2 +- lib/gitlab/pages.rb | 8 ++ spec/lib/gitlab/ci/config/entry/job_spec.rb | 16 ++++ spec/lib/gitlab/pages_spec.rb | 42 +++++++++- spec/models/ci/build_spec.rb | 78 ++++++++----------- 15 files changed, 302 insertions(+), 63 deletions(-) create mode 100644 ee/app/services/ee/projects/update_pages_service.rb create mode 100644 ee/lib/gitlab/ci/config/entry/pages_path_prefix.rb create mode 100644 ee/spec/services/ee/projects/update_pages_service_spec.rb diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 65091487c93e80..1274df4b505b40 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -2084,6 +2084,10 @@ "publish": { "description": "A path to a directory that contains the files to be published with Pages", "type": "string" + }, + "pages_path_prefix": { + "description": "The path prefix identifier for this version of pages. Allows creation of multiple versions of the same site with different path prefixes", + "type": "string" } }, "oneOf": [ diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 73594448192271..7706a8c5b3b1ad 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -388,9 +388,12 @@ def other_scheduled_actions def pages_generator? Gitlab.config.pages.enabled && - name == 'pages' + (name == 'pages' || name.start_with?('pages:')) end + # overridden on EE + def pages_path_prefix; end + def runnable? true end diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index 403f645392c237..f46d7a0a2d31ce 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -84,15 +84,9 @@ def create_stage # rubocop: enable Performance/ActiveRecordSubtransactionMethods def create_pages_deployment(artifacts_path, build) - sha256 = build.job_artifacts_archive.file_sha256 File.open(artifacts_path) do |file| - deployment = project.pages_deployments.create!( - file: file, - file_count: deployment_update.entries_count, - file_sha256: sha256, - ci_build_id: build.id, - root_directory: build.options[:publish] - ) + attributes = pages_deployment_attributes(file, build) + deployment = project.pages_deployments.create!(**attributes) break if deployment.size != file.size || deployment.file.size != file.size @@ -100,6 +94,17 @@ def create_pages_deployment(artifacts_path, build) end end + # overridden on EE + def pages_deployment_attributes(file, build) + { + file: file, + file_count: deployment_update.entries_count, + file_sha256: build.job_artifacts_archive.file_sha256, + ci_build_id: build.id, + root_directory: build.options[:publish] + } + end + def update_project_pages_deployment(deployment) project.update_pages_deployment!(deployment) DestroyPagesDeploymentsWorker.perform_in( @@ -144,3 +149,5 @@ def publish_deployed_event end end end + +::Projects::UpdatePagesService.prepend_mod diff --git a/ee/app/models/ee/ci/build.rb b/ee/app/models/ee/ci/build.rb index b28c303c17d68c..be8690467c4854 100644 --- a/ee/app/models/ee/ci/build.rb +++ b/ee/app/models/ee/ci/build.rb @@ -9,6 +9,7 @@ module Ci module Build extend ActiveSupport::Concern extend ::Gitlab::Utils::Override + include ::Gitlab::Utils::StrongMemoize VALIDATE_SCHEMA_VARIABLE_NAME = 'VALIDATE_SCHEMA' LICENSED_PARSER_FEATURES = { @@ -213,6 +214,16 @@ def resource_parent project end + override :pages_path_prefix + def pages_path_prefix + return unless pages_generator? + return unless options[:pages_path_prefix].present? + return unless ::Gitlab::Pages.multiple_versions_enabled_for?(project) + + ExpandVariables.expand(options[:pages_path_prefix], -> { simple_variables }) + end + strong_memoize_attr :pages_path_prefix + private def variables_hash diff --git a/ee/app/services/ee/projects/update_pages_service.rb b/ee/app/services/ee/projects/update_pages_service.rb new file mode 100644 index 00000000000000..b6119f6b3f1607 --- /dev/null +++ b/ee/app/services/ee/projects/update_pages_service.rb @@ -0,0 +1,16 @@ +# frozen_string_literal: true + +module EE + module Projects + module UpdatePagesService + extend ::Gitlab::Utils::Override + + override :pages_deployment_attributes + def pages_deployment_attributes(file, build) + return super unless ::Gitlab::Pages.multiple_versions_enabled_for?(build.project) + + super.merge(path_prefix: build.pages_path_prefix) + end + end + end +end diff --git a/ee/lib/ee/gitlab/ci/config/entry/job.rb b/ee/lib/ee/gitlab/ci/config/entry/job.rb index 965e9f6ad84334..288e42aa29ed3a 100644 --- a/ee/lib/ee/gitlab/ci/config/entry/job.rb +++ b/ee/lib/ee/gitlab/ci/config/entry/job.rb @@ -9,8 +9,10 @@ module Job extend ActiveSupport::Concern extend ::Gitlab::Utils::Override + EE_ALLOWED_KEYS = %i[dast_configuration secrets pages_path_prefix].freeze + prepended do - attributes :dast_configuration, :secrets + attributes :dast_configuration, :secrets, :pages_path_prefix entry :dast_configuration, ::Gitlab::Ci::Config::Entry::DastConfiguration, description: 'DAST configuration for this job', @@ -20,13 +22,18 @@ module Job description: 'Configured secrets for this job', inherit: false, metadata: { composable_class: ::Gitlab::Ci::Config::Entry::Secret } - end - - EE_ALLOWED_KEYS = %i[dast_configuration secrets].freeze - override :value - def value - super.merge({ dast_configuration: dast_configuration_value, secrets: secrets_value }.compact) + entry :pages_path_prefix, ::Gitlab::Ci::Config::Entry::PagesPathPrefix, + inherit: false, + description: \ + 'Pages path prefix identifier. ' \ + 'This allows to create multiple versions of the same site with different path prefixes' + + validations do + validates :pages_path_prefix, + absence: { message: "can only be used within a `pages` job" }, + unless: -> { pages_job? } + end end class_methods do @@ -37,6 +44,15 @@ def allowed_keys super + EE_ALLOWED_KEYS end end + + override :value + def value + super.merge({ + dast_configuration: dast_configuration_value, + secrets: secrets_value, + pages_path_prefix: pages_path_prefix + }.compact) + end end end end diff --git a/ee/lib/ee/gitlab/ci/yaml_processor/result.rb b/ee/lib/ee/gitlab/ci/yaml_processor/result.rb index e2daac1e7b6ac2..6835c7a4138dca 100644 --- a/ee/lib/ee/gitlab/ci/yaml_processor/result.rb +++ b/ee/lib/ee/gitlab/ci/yaml_processor/result.rb @@ -15,7 +15,10 @@ def build_attributes(name) super.deep_merge( { - options: { dast_configuration: job[:dast_configuration] }.compact, + options: { + dast_configuration: job[:dast_configuration], + pages_path_prefix: job[:pages_path_prefix] + }.compact, secrets: job[:secrets] }.compact ) diff --git a/ee/lib/gitlab/ci/config/entry/pages_path_prefix.rb b/ee/lib/gitlab/ci/config/entry/pages_path_prefix.rb new file mode 100644 index 00000000000000..81bf87efb522eb --- /dev/null +++ b/ee/lib/gitlab/ci/config/entry/pages_path_prefix.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the pages path prefix + # + class PagesPathPrefix < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String + end + end + end + end + end +end diff --git a/ee/spec/models/ci/build_spec.rb b/ee/spec/models/ci/build_spec.rb index ae68d2175a6639..2ba5369d6276b6 100644 --- a/ee/spec/models/ci/build_spec.rb +++ b/ee/spec/models/ci/build_spec.rb @@ -915,6 +915,38 @@ end end + describe '#pages_path_prefix', feature_category: :pages do + using RSpec::Parameterized::TableSyntax + + where(:result, :option, :pages_generator, :multiple_versions_enabled) do + nil | nil | false | false + nil | nil | false | true + nil | nil | true | false + nil | nil | true | true + nil | '' | true | true + 'foo' | 'foo' | true | true + 'master' | '$CI_COMMIT_BRANCH' | true | true + end + + with_them do + before do + job.options[:pages_path_prefix] = option + + allow(job) + .to receive(:pages_generator?) + .and_return(pages_generator) + + allow(::Gitlab::Pages) + .to receive(:multiple_versions_enabled_for?) + .and_return(multiple_versions_enabled) + end + + subject { job.pages_path_prefix } + + it { is_expected.to eq(result) } + end + end + context 'with loose foreign keys for partitioned tables' do before do create(:security_scan, build: job) diff --git a/ee/spec/services/ee/projects/update_pages_service_spec.rb b/ee/spec/services/ee/projects/update_pages_service_spec.rb new file mode 100644 index 00000000000000..571fe122fd2b75 --- /dev/null +++ b/ee/spec/services/ee/projects/update_pages_service_spec.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Projects::UpdatePagesService, feature_category: :pages do + let_it_be(:project) { create(:project, :repository) } + let_it_be(:build) { create_pages_build_and_artifacts(project) } + + subject(:service) { described_class.new(project, build) } + + before do + stub_pages_setting(enabled: true) + end + + context 'when pages_multiple_versions is not enabled for project' do + it 'does not save the given path prefix' do + expect(::Gitlab::Pages) + .to receive(:multiple_versions_enabled_for?) + .with(build.project) + .and_return(false) + + expect do + expect(service.execute[:status]).to eq(:success) + end.to change { project.pages_deployments.count }.by(1) + + deployment = project.pages_deployments.last + + expect(deployment.path_prefix).to be_nil + end + end + + context 'when pages_multiple_versions is enabled for project' do + it 'saves the given path prefix' do + expect(::Gitlab::Pages) + .to receive(:multiple_versions_enabled_for?) + .with(build.project) + .and_return(true) + .twice # it's called from within the build.pages_path_prefix as well + + expect do + expect(service.execute[:status]).to eq(:success) + end.to change { project.pages_deployments.count }.by(1) + + deployment = project.pages_deployments.last + + expect(deployment.path_prefix).to eq('__pages__prefix__') + end + end + + def create_pages_build_and_artifacts(project) + build = create( + :ci_build, + name: 'pages', + pipeline: create(:ci_pipeline, project: project, sha: project.commit('HEAD').sha), + ref: 'HEAD', + options: { pages_path_prefix: '__pages__prefix__' }) + + create( + :ci_job_artifact, + :correct_checksum, + file: fixture_file_upload('spec/fixtures/pages.zip'), + job: build) + + create( + :ci_job_artifact, + file_type: :metadata, + file_format: :gzip, + file: fixture_file_upload('spec/fixtures/pages.zip.meta'), + job: build) + + build.reload + end +end diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index d31d1b366c3fe2..9b63a2d77ba5ca 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -185,7 +185,7 @@ def ignored? end def pages_job? - name == :pages + name == :pages || name.to_s.start_with?('pages:') end def self.allowed_keys diff --git a/lib/gitlab/pages.rb b/lib/gitlab/pages.rb index 4e0e5102becbcc..367fe7639da09a 100644 --- a/lib/gitlab/pages.rb +++ b/lib/gitlab/pages.rb @@ -23,6 +23,14 @@ def access_control_is_forced? ::Gitlab.config.pages.access_control && ::Gitlab::CurrentSettings.current_application_settings.force_pages_access_control end + + def multiple_versions_enabled_for?(project) + return false if project.blank? + + ::Feature.enabled?(:pages_multiple_versions_setting, project) && + project.licensed_feature_available?(:pages_multiple_versions) && + project.project_setting.pages_multiple_versions_enabled + end end end end diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 4be7c11fab0c69..39c4436b1b50ec 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -739,6 +739,22 @@ end end + describe '#pages_job?', :aggregate_failures, feature_category: :pages do + using RSpec::Parameterized::TableSyntax + + where(:result, :name) do + true | :pages + true | :'pages:staging' + false | :'something:pages:else' + end + + with_them do + subject { described_class.new({}, name: name).pages_job? } + + it { is_expected.to eq(result) } + end + end + context 'when composed' do before do entry.compose! diff --git a/spec/lib/gitlab/pages_spec.rb b/spec/lib/gitlab/pages_spec.rb index 9f85efd56e6c60..e1a08afd2efa7d 100644 --- a/spec/lib/gitlab/pages_spec.rb +++ b/spec/lib/gitlab/pages_spec.rb @@ -2,7 +2,7 @@ require 'spec_helper' -RSpec.describe Gitlab::Pages do +RSpec.describe Gitlab::Pages, feature_category: :pages do using RSpec::Parameterized::TableSyntax let(:pages_secret) { SecureRandom.random_bytes(Gitlab::Pages::SECRET_LENGTH) } @@ -48,4 +48,44 @@ it { is_expected.to eq(result) } end end + + describe '.multiple_versions_enabled_for?' do + using RSpec::Parameterized::TableSyntax + + context 'when project is nil' do + it 'returns false' do + expect(described_class.multiple_versions_enabled_for?(nil)).to eq(false) + end + end + + context 'when a project is given' do + let_it_be(:project) { create(:project) } + + where(:result, :setting, :feature_flag, :license) do + false | false | false | false + false | false | false | true + false | false | true | false + false | false | true | true + false | true | false | false + false | true | false | true + false | true | true | false + true | true | true | true + end + + with_them do + let_it_be(:project) { create(:project) } + + subject { described_class.multiple_versions_enabled_for?(project) } + + before do + stub_licensed_features(pages_multiple_versions: license) + stub_feature_flags(pages_multiple_versions_setting: feature_flag) + project.project_setting.update!(pages_multiple_versions_enabled: setting) + end + + # this feature is only available in EE + it { is_expected.to eq(result && Gitlab.ee?) } + end + end + end end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index f17cbd821c5ff5..0076e04f9a5df3 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -4025,73 +4025,63 @@ def run_job_without_exception end end - describe 'pages deployments' do - let_it_be(:build, reload: true) { create(:ci_build, pipeline: pipeline, user: user) } + describe '#pages_generator?' do + using RSpec::Parameterized::TableSyntax + + where(:result, :name, :enabled) do + false | 'foo' | false + false | 'pages' | false + true | 'pages' | true + true | 'pages:preview' | true + end - context 'when job is "pages"' do + with_them do before do - build.name = 'pages' + stub_pages_setting(enabled: enabled) + build.update!(name: name) end - context 'when pages are enabled' do - before do - allow(Gitlab.config.pages).to receive_messages(enabled: true) - end - - it 'is marked as pages generator' do - expect(build).to be_pages_generator - end - - context 'job succeeds' do - it "calls pages worker" do - expect(PagesWorker).to receive(:perform_async).with(:deploy, build.id) + subject { build.pages_generator? } - build.success! - end - end + it { is_expected.to eq(result) } + end + end - context 'job fails' do - it "does not call pages worker" do - expect(PagesWorker).not_to receive(:perform_async) + describe 'pages deployments', feature_category: :pages do + let_it_be(:build, reload: true) { create(:ci_build, name: 'pages', pipeline: pipeline, user: user) } - build.drop! - end - end + context 'when pages are enabled' do + before do + stub_pages_setting(enabled: true) end - context 'when pages are disabled' do - before do - allow(Gitlab.config.pages).to receive_messages(enabled: false) - end + context 'and job succeeds' do + it "calls pages worker" do + expect(PagesWorker).to receive(:perform_async).with(:deploy, build.id) - it 'is not marked as pages generator' do - expect(build).not_to be_pages_generator + build.success! end + end - context 'job succeeds' do - it "does not call pages worker" do - expect(PagesWorker).not_to receive(:perform_async) + context 'and job fails' do + it "does not call pages worker" do + expect(PagesWorker).not_to receive(:perform_async) - build.success! - end + build.drop! end end end - context 'when job is not "pages"' do + context 'when pages are disabled' do before do - build.name = 'other-job' - end - - it 'is not marked as pages generator' do - expect(build).not_to be_pages_generator + stub_pages_setting(enabled: false) end - context 'job succeeds' do + context 'and job succeeds' do it "does not call pages worker" do expect(PagesWorker).not_to receive(:perform_async) - build.success + build.success! end end end -- GitLab From fb779ffb34708e8b77e0d5d8e3694afe5a45c461 Mon Sep 17 00:00:00 2001 From: Kassio Borges Date: Thu, 31 Aug 2023 14:35:34 +0200 Subject: [PATCH 2/6] Remove duplicated TableSyntax loading and improve a test truth table readability --- ee/spec/models/ci/build_spec.rb | 2 -- spec/lib/gitlab/ci/config/entry/job_spec.rb | 6 ++---- spec/lib/gitlab/pages_spec.rb | 10 ++++------ spec/models/ci/build_spec.rb | 9 ++------- 4 files changed, 8 insertions(+), 19 deletions(-) diff --git a/ee/spec/models/ci/build_spec.rb b/ee/spec/models/ci/build_spec.rb index 2ba5369d6276b6..162a7875d29aa3 100644 --- a/ee/spec/models/ci/build_spec.rb +++ b/ee/spec/models/ci/build_spec.rb @@ -916,8 +916,6 @@ end describe '#pages_path_prefix', feature_category: :pages do - using RSpec::Parameterized::TableSyntax - where(:result, :option, :pages_generator, :multiple_versions_enabled) do nil | nil | false | false nil | nil | false | true diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 39c4436b1b50ec..682367a077c07d 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -3,6 +3,8 @@ require 'spec_helper' RSpec.describe Gitlab::Ci::Config::Entry::Job, feature_category: :pipeline_composition do + using RSpec::Parameterized::TableSyntax + let(:entry) { described_class.new(config, name: :rspec) } it_behaves_like 'with inheritable CI config' do @@ -696,8 +698,6 @@ end context 'with workflow rules' do - using RSpec::Parameterized::TableSyntax - where(:name, :has_workflow_rules?, :only, :rules, :result) do "uses default only" | false | nil | nil | { refs: %w[branches tags] } "uses user only" | false | %w[branches] | nil | { refs: %w[branches] } @@ -740,8 +740,6 @@ end describe '#pages_job?', :aggregate_failures, feature_category: :pages do - using RSpec::Parameterized::TableSyntax - where(:result, :name) do true | :pages true | :'pages:staging' diff --git a/spec/lib/gitlab/pages_spec.rb b/spec/lib/gitlab/pages_spec.rb index e1a08afd2efa7d..20b7821f4d2942 100644 --- a/spec/lib/gitlab/pages_spec.rb +++ b/spec/lib/gitlab/pages_spec.rb @@ -50,8 +50,6 @@ end describe '.multiple_versions_enabled_for?' do - using RSpec::Parameterized::TableSyntax - context 'when project is nil' do it 'returns false' do expect(described_class.multiple_versions_enabled_for?(nil)).to eq(false) @@ -61,14 +59,14 @@ context 'when a project is given' do let_it_be(:project) { create(:project) } - where(:result, :setting, :feature_flag, :license) do + where(:setting, :feature_flag, :license, :result) do false | false | false | false - false | false | false | true false | false | true | false - false | false | true | true false | true | false | false - false | true | false | true false | true | true | false + true | false | false | false + true | false | true | false + true | true | false | false true | true | true | true end diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 0076e04f9a5df3..0118bd84567eb5 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -3,6 +3,7 @@ require 'spec_helper' RSpec.describe Ci::Build, feature_category: :continuous_integration, factory_default: :keep do + using RSpec::Parameterized::TableSyntax include Ci::TemplateHelpers include AfterNextHelpers @@ -1493,8 +1494,6 @@ end describe 'state transition metrics' do - using RSpec::Parameterized::TableSyntax - subject { build.send(event) } where(:state, :report_count, :trait) do @@ -2129,8 +2128,6 @@ end describe '#ref_slug' do - using RSpec::Parameterized::TableSyntax - where(:ref, :slug) do 'master' | 'master' '1-foo' | '1-foo' @@ -4025,9 +4022,7 @@ def run_job_without_exception end end - describe '#pages_generator?' do - using RSpec::Parameterized::TableSyntax - + describe '#pages_generator?', feature_category: :pages do where(:result, :name, :enabled) do false | 'foo' | false false | 'pages' | false -- GitLab From 958ba3dde6e7b037c70f4366bbfa124596c772e5 Mon Sep 17 00:00:00 2001 From: Kassio Borges Date: Thu, 31 Aug 2023 18:59:12 +0200 Subject: [PATCH 3/6] Add missing tests --- .../ci/config/entry/pages_path_prefix_spec.rb | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) create mode 100644 ee/spec/lib/gitlab/ci/config/entry/pages_path_prefix_spec.rb diff --git a/ee/spec/lib/gitlab/ci/config/entry/pages_path_prefix_spec.rb b/ee/spec/lib/gitlab/ci/config/entry/pages_path_prefix_spec.rb new file mode 100644 index 00000000000000..f63e303b199edb --- /dev/null +++ b/ee/spec/lib/gitlab/ci/config/entry/pages_path_prefix_spec.rb @@ -0,0 +1,33 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Ci::Config::Entry::PagesPathPrefix, :aggregate_failures, feature_category: :pages do + let(:entry) { described_class.new(config) } + + describe 'validations' do + context 'when config value is correct' do + let(:config) { 'prefix' } + + describe '#config' do + it 'returns the given value' do + expect(entry.config).to eq config + end + end + + describe '#valid?' do + it 'is valid' do + expect(entry).to be_valid + end + end + end + + context 'when the value has a wrong type' do + let(:config) { { test: true } } + + it 'reports an error' do + expect(entry.errors).to include 'pages path prefix config should be a string' + end + end + end +end -- GitLab From f5cf0ebeab9775e9cc0d524413bd2a650a9ea07a Mon Sep 17 00:00:00 2001 From: Kassio Borges Date: Sat, 2 Sep 2023 18:30:55 +0200 Subject: [PATCH 4/6] Use "#{build.name}:deploy" for pages deploy jobs --- app/services/projects/update_pages_service.rb | 27 +-- .../projects/update_pages_service_spec.rb | 159 +++++++++++++----- .../pages_size_limit_shared_examples.rb | 32 ---- 3 files changed, 135 insertions(+), 83 deletions(-) delete mode 100644 spec/support/shared_examples/services/pages_size_limit_shared_examples.rb diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index f46d7a0a2d31ce..fa1b57efb364ea 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -2,6 +2,8 @@ module Projects class UpdatePagesService < BaseService + include Gitlab::Utils::StrongMemoize + # old deployment can be cached by pages daemon # so we need to give pages daemon some time update cache # 10 minutes is enough, but 30 feels safer @@ -16,11 +18,11 @@ def initialize(project, build) end def execute + return error('Build is not a pages generator') unless build.pages_generator? + register_attempt - # Create status notifying the deployment of pages - @commit_status = build_commit_status - ::Ci::Pipelines::AddJobService.new(@build.pipeline).execute!(@commit_status) do |job| + ::Ci::Pipelines::AddJobService.new(@build.pipeline).execute!(commit_status) do |job| job.enqueue! job.run! end @@ -47,7 +49,7 @@ def execute private def success - @commit_status.success + commit_status.success @project.mark_pages_as_deployed publish_deployed_event super @@ -56,31 +58,32 @@ def success def error(message) register_failure log_error("Projects::UpdatePagesService: #{message}") - @commit_status.allow_failure = !deployment_update.latest? - @commit_status.description = message - @commit_status.drop(:script_failure) + commit_status.allow_failure = !deployment_update.latest? + commit_status.description = message + commit_status.drop(:script_failure) super end - def build_commit_status - stage = create_stage - + # Create status notifying the deployment of pages + def commit_status GenericCommitStatus.new( user: build.user, ci_stage: stage, - name: 'pages:deploy', + name: "#{build.name}:deploy", stage: 'deploy', stage_idx: stage.position ) end + strong_memoize_attr :commit_status # rubocop: disable Performance/ActiveRecordSubtransactionMethods - def create_stage + def stage build.pipeline.stages.safe_find_or_create_by(name: 'deploy', pipeline_id: build.pipeline.id) do |stage| stage.position = GenericCommitStatus::EXTERNAL_STAGE_IDX stage.project = build.project end end + strong_memoize_attr :commit_status # rubocop: enable Performance/ActiveRecordSubtransactionMethods def create_pages_deployment(artifacts_path, build) diff --git a/spec/services/projects/update_pages_service_spec.rb b/spec/services/projects/update_pages_service_spec.rb index a113f3506e13cc..b2e5e272d0e2f8 100644 --- a/spec/services/projects/update_pages_service_spec.rb +++ b/spec/services/projects/update_pages_service_spec.rb @@ -9,7 +9,8 @@ let_it_be(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit('HEAD').sha) } let(:options) { {} } - let(:build) { create(:ci_build, pipeline: pipeline, ref: 'HEAD', options: options) } + let(:build_name) { 'pages' } + let(:build) { create(:ci_build, name: build_name, pipeline: pipeline, ref: 'HEAD', options: options) } let(:invalid_file) { fixture_file_upload('spec/fixtures/dk.png') } let(:file) { fixture_file_upload("spec/fixtures/pages.zip") } @@ -20,13 +21,60 @@ let(:custom_root_file_metadata) { "spec/fixtures/pages_with_custom_root.zip.meta" } let(:metadata) { fixture_file_upload(metadata_filename) if File.exist?(metadata_filename) } - subject { described_class.new(project, build) } + subject(:service) { described_class.new(project, build) } + + before do + stub_pages_setting(enabled: true) + end + + RSpec.shared_examples 'pages size limit is' do |size_limit| + context "when size is below the limit" do + before do + allow(metadata).to receive(:total_size).and_return(size_limit - 1.megabyte) + allow(metadata).to receive(:entries).and_return([]) + end + + it 'updates pages correctly' do + subject.execute + + deploy_status = GenericCommitStatus.last + expect(deploy_status.description).not_to be_present + expect(project.pages_metadatum).to be_deployed + end + end + + context "when size is above the limit" do + before do + allow(metadata).to receive(:total_size).and_return(size_limit + 1.megabyte) + allow(metadata).to receive(:entries).and_return([]) + end + + it 'limits the maximum size of gitlab pages' do + subject.execute + + deploy_status = GenericCommitStatus.last + expect(deploy_status.description).to match(/artifacts for pages are too large/) + expect(deploy_status).to be_script_failure + end + end + end + + context 'when build is not a pages generator' do + it 'returns an error' do + stub_pages_setting(enabled: false) + + result = service.execute + + expect(result[:status]).to eq(:error) + expect(result[:message]).to eq('Build is not a pages generator') + end + end context 'when a deploy stage already exists', :aggregate_failures do let!(:stage) { create(:ci_stage, name: 'deploy', pipeline: pipeline) } it 'assigns the deploy stage' do - expect { subject.execute } + expect { service.execute } .to change(GenericCommitStatus, :count).by(1) .and change(Ci::Stage.where(name: 'deploy'), :count).by(0) @@ -36,12 +84,30 @@ expect(status.ci_stage.name).to eq('deploy') expect(status.stage_name).to eq('deploy') expect(status.stage).to eq('deploy') + expect(status.name).to eq('pages:deploy') + end + + context 'when the build name is pages:deploy' do + let(:build_name) { 'pages:deploy' } + + it 'assigns the deploy stage' do + expect { service.execute } + .to change(GenericCommitStatus, :count).by(1) + .and change(Ci::Stage.where(name: 'deploy'), :count).by(0) + + status = GenericCommitStatus.last + + expect(status.ci_stage).to eq(stage) + expect(status.stage_name).to eq('deploy') + expect(status.stage).to eq('deploy') + expect(status.name).to eq('pages:deploy:deploy') + end end end context 'when a deploy stage does not exists' do it 'assigns the deploy stage' do - expect { subject.execute } + expect { service.execute } .to change(GenericCommitStatus, :count).by(1) .and change(Ci::Stage.where(name: 'deploy'), :count).by(1) @@ -50,6 +116,24 @@ expect(status.ci_stage.name).to eq('deploy') expect(status.stage_name).to eq('deploy') expect(status.stage).to eq('deploy') + expect(status.name).to eq('pages:deploy') + end + + context 'when the build name is pages:deploy' do + let(:build_name) { 'pages:deploy' } + + it 'assigns the deploy stage' do + expect { service.execute } + .to change(GenericCommitStatus, :count).by(1) + .and change(Ci::Stage.where(name: 'deploy'), :count).by(1) + + status = GenericCommitStatus.last + + expect(status.ci_stage.name).to eq('deploy') + expect(status.stage_name).to eq('deploy') + expect(status.stage).to eq('deploy') + expect(status.name).to eq('pages:deploy:deploy') + end end end @@ -64,7 +148,7 @@ end it "doesn't delete artifacts after deploying" do - expect(execute).to eq(:success) + expect(service.execute[:status]).to eq(:success) expect(project.pages_metadatum).to be_deployed expect(build.artifacts?).to eq(true) @@ -72,7 +156,7 @@ it 'succeeds' do expect(project.pages_deployed?).to be_falsey - expect(execute).to eq(:success) + expect(service.execute[:status]).to eq(:success) expect(project.pages_metadatum).to be_deployed expect(project.pages_deployed?).to be_truthy end @@ -84,12 +168,12 @@ root_namespace_id: project.root_namespace.id } - expect { subject.execute }.to publish_event(Pages::PageDeployedEvent).with(expected_data) + expect { service.execute }.to publish_event(Pages::PageDeployedEvent).with(expected_data) end it 'creates pages_deployment and saves it in the metadata' do expect do - expect(execute).to eq(:success) + expect(service.execute[:status]).to eq(:success) end.to change { project.pages_deployments.count }.by(1) deployment = project.pages_deployments.last @@ -108,7 +192,7 @@ project.reload expect do - expect(execute).to eq(:success) + expect(service.execute[:status]).to eq(:success) end.to change { project.pages_deployments.count }.by(1) expect(project.pages_metadatum.reload.pages_deployment).to eq(project.pages_deployments.last) @@ -127,12 +211,12 @@ ) ) - execute + service.execute end it 'removes older deployments', :sidekiq_inline do expect do - execute + service.execute end.not_to change { PagesDeployment.count } # it creates one and deletes one expect(PagesDeployment.find_by_id(old_deployment.id)).to be_nil @@ -144,7 +228,7 @@ let(:metadata_filename) { empty_metadata_filename } it 'returns an error' do - expect(execute).not_to eq(:success) + expect(service.execute[:status]).not_to eq(:success) expect(GenericCommitStatus.last.description).to eq("Error: You need to either include a `public/` folder in your artifacts, or specify which one to use for Pages using `publish` in `.gitlab-ci.yml`") end @@ -158,7 +242,7 @@ let(:options) { { publish: 'foo' } } it 'creates pages_deployment and saves it in the metadata' do - expect(execute).to eq(:success) + expect(service.execute[:status]).to eq(:success) deployment = project.pages_deployments.last expect(deployment.root_directory).to eq(options[:publish]) @@ -169,7 +253,7 @@ let(:options) { { publish: 'bar' } } it 'returns an error' do - expect(execute).not_to eq(:success) + expect(service.execute[:status]).not_to eq(:success) expect(GenericCommitStatus.last.description).to eq("Error: You need to either include a `public/` folder in your artifacts, or specify which one to use for Pages using `publish` in `.gitlab-ci.yml`") end @@ -181,7 +265,7 @@ let(:metadata_filename) { "spec/fixtures/pages.zip.meta" } it 'returns an error' do - expect(execute).not_to eq(:success) + expect(service.execute[:status]).not_to eq(:success) expect(GenericCommitStatus.last.description).to eq("Error: You need to either include a `public/` folder in your artifacts, or specify which one to use for Pages using `publish` in `.gitlab-ci.yml`") end @@ -190,13 +274,13 @@ it 'limits pages size' do stub_application_setting(max_pages_size: 1) - expect(execute).not_to eq(:success) + expect(service.execute[:status]).not_to eq(:success) end it 'limits pages file count' do create(:plan_limits, :default_plan, pages_file_entries: 2) - expect(execute).not_to eq(:success) + expect(service.execute[:status]).not_to eq(:success) expect(GenericCommitStatus.last.description).to eq("pages site contains 3 file entries, while limit is set to 2") end @@ -209,9 +293,11 @@ end it 'raises an error' do - expect { execute }.to raise_error(SocketError) + expect { service.execute }.to raise_error(SocketError) build.reload + + deploy_status = GenericCommitStatus.last expect(deploy_status).to be_failed expect(project.pages_metadatum).not_to be_deployed end @@ -223,9 +309,11 @@ end it 'does not raise an error as failed job' do - execute + service.execute build.reload + + deploy_status = GenericCommitStatus.last expect(deploy_status).to be_failed expect(project.pages_metadatum).not_to be_deployed end @@ -234,7 +322,7 @@ context 'with background jobs running', :sidekiq_inline do it 'succeeds' do expect(project.pages_deployed?).to be_falsey - expect(execute).to eq(:success) + expect(service.execute[:status]).to eq(:success) end end @@ -249,7 +337,7 @@ shared_examples 'successfully deploys' do it 'succeeds' do expect do - expect(execute).to eq(:success) + expect(service.execute[:status]).to eq(:success) end.to change { project.pages_deployments.count }.by(1) deployment = project.pages_deployments.last @@ -261,7 +349,7 @@ context 'when old deployment present' do before do - old_build = create(:ci_build, pipeline: old_pipeline, ref: 'HEAD') + old_build = create(:ci_build, name: 'pages', pipeline: old_pipeline, ref: 'HEAD') old_deployment = create(:pages_deployment, ci_build: old_build, project: project) project.update_pages_deployment!(old_deployment) end @@ -272,15 +360,16 @@ context 'when newer deployment present' do before do new_pipeline = create(:ci_pipeline, project: project, sha: project.commit('HEAD').sha) - new_build = create(:ci_build, pipeline: new_pipeline, ref: 'HEAD') + new_build = create(:ci_build, name: 'pages', pipeline: new_pipeline, ref: 'HEAD') new_deployment = create(:pages_deployment, ci_build: new_build, project: project) project.update_pages_deployment!(new_deployment) end it 'fails with outdated reference message' do - expect(execute).to eq(:error) + expect(service.execute[:status]).to eq(:error) expect(project.reload.pages_metadatum).not_to be_deployed + deploy_status = GenericCommitStatus.last expect(deploy_status).to be_failed expect(deploy_status.description).to eq('build SHA is outdated for this ref') end @@ -294,7 +383,7 @@ .and_return(file.size + 1) end - expect(execute).not_to eq(:success) + expect(service.execute[:status]).not_to eq(:success) expect(GenericCommitStatus.last.description).to eq('The uploaded artifact size does not match the expected value') project.pages_metadatum.reload @@ -318,18 +407,18 @@ it 'fails with exception raised' do expect do - execute + service.execute end.to raise_error("Validation failed: File sha256 can't be blank") end end it 'fails if no artifacts' do - expect(execute).not_to eq(:success) + expect(service.execute[:status]).not_to eq(:success) end it 'fails for invalid archive' do create(:ci_job_artifact, :archive, file: invalid_file, job: build) - expect(execute).not_to eq(:success) + expect(service.execute[:status]).not_to eq(:success) end describe 'maximum pages artifacts size' do @@ -383,21 +472,13 @@ end it 'marks older pages:deploy jobs retried' do - expect(execute).to eq(:success) + expect(service.execute[:status]).to eq(:success) expect(older_deploy_job.reload).to be_retried + + deploy_status = GenericCommitStatus.last expect(deploy_status.ci_stage).to eq(stage) expect(deploy_status.stage_idx).to eq(stage.position) end end - - private - - def deploy_status - GenericCommitStatus.where(name: 'pages:deploy').last - end - - def execute - subject.execute[:status] - end end diff --git a/spec/support/shared_examples/services/pages_size_limit_shared_examples.rb b/spec/support/shared_examples/services/pages_size_limit_shared_examples.rb deleted file mode 100644 index d9e906ebb75d8f..00000000000000 --- a/spec/support/shared_examples/services/pages_size_limit_shared_examples.rb +++ /dev/null @@ -1,32 +0,0 @@ -# frozen_string_literal: true - -RSpec.shared_examples 'pages size limit is' do |size_limit| - context "when size is below the limit" do - before do - allow(metadata).to receive(:total_size).and_return(size_limit - 1.megabyte) - allow(metadata).to receive(:entries).and_return([]) - end - - it 'updates pages correctly' do - subject.execute - - expect(deploy_status.description).not_to be_present - expect(project.pages_metadatum).to be_deployed - end - end - - context "when size is above the limit" do - before do - allow(metadata).to receive(:total_size).and_return(size_limit + 1.megabyte) - allow(metadata).to receive(:entries).and_return([]) - end - - it 'limits the maximum size of gitlab pages' do - subject.execute - - expect(deploy_status.description) - .to match(/artifacts for pages are too large/) - expect(deploy_status).to be_script_failure - end - end -end -- GitLab From 4ec0ead997890c6c0f365b2adfd99fe1411822a4 Mon Sep 17 00:00:00 2001 From: Kassio Borges Date: Sun, 3 Sep 2023 20:24:59 +0200 Subject: [PATCH 5/6] Remove the pages: prefix --- app/models/ci/build.rb | 2 +- app/services/projects/update_pages_service.rb | 4 +- lib/gitlab/ci/config/entry/job.rb | 2 +- spec/lib/gitlab/ci/config/entry/job_spec.rb | 8 +-- spec/models/ci/build_spec.rb | 10 ++-- .../projects/update_pages_service_spec.rb | 54 +------------------ 6 files changed, 13 insertions(+), 67 deletions(-) diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index 7706a8c5b3b1ad..107f3c5987a68c 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -388,7 +388,7 @@ def other_scheduled_actions def pages_generator? Gitlab.config.pages.enabled && - (name == 'pages' || name.start_with?('pages:')) + name == 'pages' end # overridden on EE diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index fa1b57efb364ea..7d25e4cff4d66a 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -18,8 +18,6 @@ def initialize(project, build) end def execute - return error('Build is not a pages generator') unless build.pages_generator? - register_attempt ::Ci::Pipelines::AddJobService.new(@build.pipeline).execute!(commit_status) do |job| @@ -69,7 +67,7 @@ def commit_status GenericCommitStatus.new( user: build.user, ci_stage: stage, - name: "#{build.name}:deploy", + name: 'pages:deploy', stage: 'deploy', stage_idx: stage.position ) diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index 9b63a2d77ba5ca..d31d1b366c3fe2 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -185,7 +185,7 @@ def ignored? end def pages_job? - name == :pages || name.to_s.start_with?('pages:') + name == :pages end def self.allowed_keys diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 682367a077c07d..6c078f3dc0f48d 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -740,10 +740,10 @@ end describe '#pages_job?', :aggregate_failures, feature_category: :pages do - where(:result, :name) do - true | :pages - true | :'pages:staging' - false | :'something:pages:else' + where(:name, :result) do + :pages | true + :'pages:staging' | false + :'something:pages:else' | false end with_them do diff --git a/spec/models/ci/build_spec.rb b/spec/models/ci/build_spec.rb index 0118bd84567eb5..b95dec7195ffac 100644 --- a/spec/models/ci/build_spec.rb +++ b/spec/models/ci/build_spec.rb @@ -4023,11 +4023,11 @@ def run_job_without_exception end describe '#pages_generator?', feature_category: :pages do - where(:result, :name, :enabled) do - false | 'foo' | false - false | 'pages' | false - true | 'pages' | true - true | 'pages:preview' | true + where(:name, :enabled, :result) do + 'foo' | false | false + 'pages' | false | false + 'pages:preview' | true | false + 'pages' | true | true end with_them do diff --git a/spec/services/projects/update_pages_service_spec.rb b/spec/services/projects/update_pages_service_spec.rb index b2e5e272d0e2f8..a250be0c94f6e4 100644 --- a/spec/services/projects/update_pages_service_spec.rb +++ b/spec/services/projects/update_pages_service_spec.rb @@ -9,8 +9,7 @@ let_it_be(:pipeline) { create(:ci_pipeline, project: project, sha: project.commit('HEAD').sha) } let(:options) { {} } - let(:build_name) { 'pages' } - let(:build) { create(:ci_build, name: build_name, pipeline: pipeline, ref: 'HEAD', options: options) } + let(:build) { create(:ci_build, pipeline: pipeline, ref: 'HEAD', options: options) } let(:invalid_file) { fixture_file_upload('spec/fixtures/dk.png') } let(:file) { fixture_file_upload("spec/fixtures/pages.zip") } @@ -23,10 +22,6 @@ subject(:service) { described_class.new(project, build) } - before do - stub_pages_setting(enabled: true) - end - RSpec.shared_examples 'pages size limit is' do |size_limit| context "when size is below the limit" do before do @@ -59,17 +54,6 @@ end end - context 'when build is not a pages generator' do - it 'returns an error' do - stub_pages_setting(enabled: false) - - result = service.execute - - expect(result[:status]).to eq(:error) - expect(result[:message]).to eq('Build is not a pages generator') - end - end - context 'when a deploy stage already exists', :aggregate_failures do let!(:stage) { create(:ci_stage, name: 'deploy', pipeline: pipeline) } @@ -84,24 +68,6 @@ expect(status.ci_stage.name).to eq('deploy') expect(status.stage_name).to eq('deploy') expect(status.stage).to eq('deploy') - expect(status.name).to eq('pages:deploy') - end - - context 'when the build name is pages:deploy' do - let(:build_name) { 'pages:deploy' } - - it 'assigns the deploy stage' do - expect { service.execute } - .to change(GenericCommitStatus, :count).by(1) - .and change(Ci::Stage.where(name: 'deploy'), :count).by(0) - - status = GenericCommitStatus.last - - expect(status.ci_stage).to eq(stage) - expect(status.stage_name).to eq('deploy') - expect(status.stage).to eq('deploy') - expect(status.name).to eq('pages:deploy:deploy') - end end end @@ -116,24 +82,6 @@ expect(status.ci_stage.name).to eq('deploy') expect(status.stage_name).to eq('deploy') expect(status.stage).to eq('deploy') - expect(status.name).to eq('pages:deploy') - end - - context 'when the build name is pages:deploy' do - let(:build_name) { 'pages:deploy' } - - it 'assigns the deploy stage' do - expect { service.execute } - .to change(GenericCommitStatus, :count).by(1) - .and change(Ci::Stage.where(name: 'deploy'), :count).by(1) - - status = GenericCommitStatus.last - - expect(status.ci_stage.name).to eq('deploy') - expect(status.stage_name).to eq('deploy') - expect(status.stage).to eq('deploy') - expect(status.name).to eq('pages:deploy:deploy') - end end end -- GitLab From 5f8802f7b6416d30ee1412e8103f7b13b8897dae Mon Sep 17 00:00:00 2001 From: Kassio Borges Date: Sun, 3 Sep 2023 22:19:40 +0200 Subject: [PATCH 6/6] Fix indentation (rubocop) --- spec/lib/gitlab/ci/config/entry/job_spec.rb | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/lib/gitlab/ci/config/entry/job_spec.rb b/spec/lib/gitlab/ci/config/entry/job_spec.rb index 6c078f3dc0f48d..980ea2e2765b27 100644 --- a/spec/lib/gitlab/ci/config/entry/job_spec.rb +++ b/spec/lib/gitlab/ci/config/entry/job_spec.rb @@ -741,9 +741,9 @@ describe '#pages_job?', :aggregate_failures, feature_category: :pages do where(:name, :result) do - :pages | true - :'pages:staging' | false - :'something:pages:else' | false + :pages | true + :'pages:staging' | false + :'something:pages:else' | false end with_them do -- GitLab