From 9651db9532bddfeaf93fb6c406e16201525ebe73 Mon Sep 17 00:00:00 2001 From: Kassio Borges Date: Mon, 19 Jun 2023 10:41:21 +0100 Subject: [PATCH] PoC: Add multiple version support for PagesDeployments This will enabled users to have: - Multiple versions of the same pages site - Preview MR changes on pages Related to: https://gitlab.com/gitlab-org/gitlab/-/issues/407776 --- app/assets/javascripts/editor/schema/ci.json | 4 ++++ app/models/ci/build.rb | 9 +++++++- app/models/namespace.rb | 9 ++++++++ app/models/pages/lookup_path.rb | 16 +++++++------ app/models/pages/virtual_domain.rb | 23 ++++++++++++------- app/models/project.rb | 6 ++--- app/services/projects/update_pages_service.rb | 4 +++- ..._prefix_and_branch_to_pages_deployments.rb | 15 ++++++++++++ ..._prefix_and_branch_on_pages_deployments.rb | 17 ++++++++++++++ db/schema_migrations/20230619093306 | 1 + db/schema_migrations/20230619093307 | 1 + db/structure.sql | 4 ++++ lib/gitlab/ci/config/entry/job.rb | 22 ++++++++++++++---- lib/gitlab/ci/config/entry/pages_prefix.rb | 20 ++++++++++++++++ lib/gitlab/ci/yaml_processor/result.rb | 3 ++- lib/gitlab/pages/virtual_host_finder.rb | 2 +- 16 files changed, 128 insertions(+), 28 deletions(-) create mode 100644 db/migrate/20230619093306_add_prefix_and_branch_to_pages_deployments.rb create mode 100644 db/migrate/20230619093307_add_limit_prefix_and_branch_on_pages_deployments.rb create mode 100644 db/schema_migrations/20230619093306 create mode 100644 db/schema_migrations/20230619093307 create mode 100644 lib/gitlab/ci/config/entry/pages_prefix.rb diff --git a/app/assets/javascripts/editor/schema/ci.json b/app/assets/javascripts/editor/schema/ci.json index 3a1188d7aabc37..c4511819037632 100644 --- a/app/assets/javascripts/editor/schema/ci.json +++ b/app/assets/javascripts/editor/schema/ci.json @@ -2018,6 +2018,10 @@ "publish": { "description": "A path to a directory that contains the files to be published with Pages", "type": "string" + }, + "pages_prefix": { + "description": "The path prefix. This allows to create multiple versions of the same site with different prefixes", + "type": "string" } }, "oneOf": [ diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index bb1bfe8c889298..07171d09bc3ff5 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -410,9 +410,16 @@ def other_scheduled_actions def pages_generator? Gitlab.config.pages.enabled && - name == 'pages' + (name == 'pages' || name.start_with?('pages:')) end + def pages_prefix + if options[:pages_prefix].present? + ExpandVariables.expand(options[:pages_prefix], -> { simple_variables }) + end + end + strong_memoize_attr :pages_prefix + def runnable? true end diff --git a/app/models/namespace.rb b/app/models/namespace.rb index 7b3bb04da5b4d9..b9da0a4eb3a2a2 100644 --- a/app/models/namespace.rb +++ b/app/models/namespace.rb @@ -606,6 +606,15 @@ def allow_runner_registration_token? !!namespace_settings&.allow_runner_registration_token? end + def all_projects_with_pages_deployments + all_projects.with_pages_deployments.distinct.includes( + :route, + :project_setting, + :project_feature, + pages_metadatum: :pages_deployment + ) + end + def all_projects_with_pages all_projects.with_pages_deployed.includes( :route, diff --git a/app/models/pages/lookup_path.rb b/app/models/pages/lookup_path.rb index 864ea04c019031..23ff5d891aa3c1 100644 --- a/app/models/pages/lookup_path.rb +++ b/app/models/pages/lookup_path.rb @@ -6,10 +6,11 @@ class LookupPath LegacyStorageDisabledError = Class.new(::StandardError) - def initialize(project, trim_prefix: nil, domain: nil) + def initialize(project, trim_prefix: nil, domain: nil, deployment: nil) @project = project @domain = domain @trim_prefix = trim_prefix || project.full_path + @deployment = deployment || project.pages_metadatum.pages_deployment end def project_id @@ -47,9 +48,9 @@ def source def prefix if project.pages_namespace_url == project.pages_url - '/' + "#{deployment.prefix}/" else - "#{project.full_path.delete_prefix(trim_prefix)}/" + prefix_path(project.full_path.delete_prefix(trim_prefix)) end end strong_memoize_attr :prefix @@ -70,11 +71,12 @@ def root_directory private - attr_reader :project, :trim_prefix, :domain + attr_reader :project, :trim_prefix, :domain, :deployment - def deployment - project.pages_metadatum.pages_deployment + def prefix_path(path) + return "#{path}/" if deployment.prefix.blank? + + "/#{[deployment.prefix.presence, path].compact.join}/" end - strong_memoize_attr :deployment end end diff --git a/app/models/pages/virtual_domain.rb b/app/models/pages/virtual_domain.rb index fafbe449c8c982..d25aaa69cc4433 100644 --- a/app/models/pages/virtual_domain.rb +++ b/app/models/pages/virtual_domain.rb @@ -18,14 +18,11 @@ def key end def lookup_paths - paths = projects.map do |project| - project.pages_lookup_path(trim_prefix: trim_prefix, domain: domain) - end - - # TODO: remove in https://gitlab.com/gitlab-org/gitlab/-/issues/328715 - paths = paths.select(&:source) - - paths.sort_by(&:prefix).reverse + projects + .flat_map { |project| pages_lookup_paths(project) } + .select(&:source) # TODO: remove in https://gitlab.com/gitlab-org/gitlab/-/issues/328715 + .sort_by { |lookup| lookup.prefix.length } + .reverse end # cache_key is required by #present_cached in ::API::Internal::Pages @@ -36,5 +33,15 @@ def cache_key private attr_reader :projects, :trim_prefix, :domain, :cache + + def pages_lookup_paths(project) + project.pages_deployments.map do |deployment| + Pages::LookupPath.new( + project, + trim_prefix: @trim_prefix, + domain: @domain, + deployment: deployment) + end + end end end diff --git a/app/models/project.rb b/app/models/project.rb index 452a5c8973c395..4e31f95f407deb 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -743,6 +743,8 @@ def self.integration_association_name(name) joins(:pages_metadatum).merge(ProjectPagesMetadatum.deployed) end + scope :with_pages_deployments, -> { joins(:pages_deployments) } + scope :pages_metadata_not_migrated, -> do left_outer_joins(:pages_metadatum) .where(project_pages_metadata: { project_id: nil }) @@ -2862,10 +2864,6 @@ def access_request_approvers_to_be_notified recipients end - def pages_lookup_path(trim_prefix: nil, domain: nil) - Pages::LookupPath.new(self, trim_prefix: trim_prefix, domain: domain) - end - def closest_setting(name) setting = read_attribute(name) setting = closest_namespace_setting(name) if setting.nil? diff --git a/app/services/projects/update_pages_service.rb b/app/services/projects/update_pages_service.rb index 403f645392c237..9058fd811f6f7e 100644 --- a/app/services/projects/update_pages_service.rb +++ b/app/services/projects/update_pages_service.rb @@ -91,7 +91,9 @@ def create_pages_deployment(artifacts_path, build) file_count: deployment_update.entries_count, file_sha256: sha256, ci_build_id: build.id, - root_directory: build.options[:publish] + root_directory: build.options[:publish], + prefix: build.pages_prefix, + branch: build.ref ) break if deployment.size != file.size || deployment.file.size != file.size diff --git a/db/migrate/20230619093306_add_prefix_and_branch_to_pages_deployments.rb b/db/migrate/20230619093306_add_prefix_and_branch_to_pages_deployments.rb new file mode 100644 index 00000000000000..2ac6e565dae8d3 --- /dev/null +++ b/db/migrate/20230619093306_add_prefix_and_branch_to_pages_deployments.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddPrefixAndBranchToPagesDeployments < Gitlab::Database::Migration[2.1] + TABLE_NAME = :pages_deployments + + # rubocop:disable Migration/AddLimitToTextColumns + # limit is added in 20230619093307_add_version_limit_to_pages_deployments + def change + add_column TABLE_NAME, :prefix, :text + add_column TABLE_NAME, :branch, :text + + add_index(TABLE_NAME, [:project_id, :prefix], unique: true) + end + # rubocop:enable Migration/AddLimitToTextColumns +end diff --git a/db/migrate/20230619093307_add_limit_prefix_and_branch_on_pages_deployments.rb b/db/migrate/20230619093307_add_limit_prefix_and_branch_on_pages_deployments.rb new file mode 100644 index 00000000000000..775083e2b10309 --- /dev/null +++ b/db/migrate/20230619093307_add_limit_prefix_and_branch_on_pages_deployments.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +class AddLimitPrefixAndBranchOnPagesDeployments < Gitlab::Database::Migration[2.1] + disable_ddl_transaction! + + TABLE_NAME = :pages_deployments + + def up + add_text_limit TABLE_NAME, :prefix, 128 + add_text_limit TABLE_NAME, :branch, 512 + end + + def down + remove_text_limit TABLE_NAME, :prefix + remove_text_limit TABLE_NAME, :branch + end +end diff --git a/db/schema_migrations/20230619093306 b/db/schema_migrations/20230619093306 new file mode 100644 index 00000000000000..0afcb1044667b4 --- /dev/null +++ b/db/schema_migrations/20230619093306 @@ -0,0 +1 @@ +926243f8f750618cf1478de41c862e796016fb637625f9ec8bd594bed435e68d \ No newline at end of file diff --git a/db/schema_migrations/20230619093307 b/db/schema_migrations/20230619093307 new file mode 100644 index 00000000000000..a9e7a1b6537548 --- /dev/null +++ b/db/schema_migrations/20230619093307 @@ -0,0 +1 @@ +3a0e18e932aae704602d7665a21c0942a2700f0bd53e6a411f1fa152dde7fc73 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 77b32db0edac9e..a4ec9d38dd4d50 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -19955,8 +19955,12 @@ CREATE TABLE pages_deployments ( file_sha256 bytea NOT NULL, size bigint, root_directory text DEFAULT 'public'::text, + prefix text, + branch text, + CONSTRAINT check_28f71e7b77 CHECK ((char_length(prefix) <= 128)), CONSTRAINT check_5f9132a958 CHECK ((size IS NOT NULL)), CONSTRAINT check_7e938c810a CHECK ((char_length(root_directory) <= 255)), + CONSTRAINT check_d9651052ec CHECK ((char_length(branch) <= 512)), CONSTRAINT check_f0fe8032dd CHECK ((char_length(file) <= 255)) ); diff --git a/lib/gitlab/ci/config/entry/job.rb b/lib/gitlab/ci/config/entry/job.rb index d31d1b366c3fe2..421f77c65c7884 100644 --- a/lib/gitlab/ci/config/entry/job.rb +++ b/lib/gitlab/ci/config/entry/job.rb @@ -14,7 +14,7 @@ class Job < ::Gitlab::Config::Entry::Node ALLOWED_KEYS = %i[tags script image services start_in artifacts cache dependencies before_script after_script hooks environment coverage retry parallel interruptible timeout - release id_tokens publish].freeze + release id_tokens publish pages_prefix].freeze validations do validates :config, allowed_keys: Gitlab::Ci::Config::Entry::Job.allowed_keys + PROCESSABLE_ALLOWED_KEYS @@ -46,7 +46,13 @@ class Job < ::Gitlab::Config::Entry::Node end end - validates :publish, absence: { message: "can only be used within a `pages` job" }, unless: -> { pages_job? } + validates :publish, + absence: { message: "can only be used within a `pages` job" }, + unless: -> { pages_job? } + + validates :pages_prefix, + absence: { message: "can only be used within a `pages` job" }, + unless: -> { pages_job? } end entry :before_script, Entry::Commands, @@ -131,10 +137,15 @@ class Job < ::Gitlab::Config::Entry::Node description: 'Path to be published with Pages', inherit: false + entry :pages_prefix, Entry::PagesPrefix, + description: 'The path prefix. This allows to create multiple versions of the same site with different prefixes', + inherit: false + attributes :script, :tags, :when, :dependencies, :needs, :retry, :parallel, :start_in, :interruptible, :timeout, - :release, :allow_failure, :publish + :release, :allow_failure, :publish, + :pages_prefix def self.matching?(name, config) !name.to_s.start_with?('.') && @@ -176,7 +187,8 @@ def value needs: needs_defined? ? needs_value : nil, scheduling_type: needs_defined? ? :dag : :stage, id_tokens: id_tokens_value, - publish: publish + publish: publish, + pages_prefix: pages_prefix ).compact end @@ -185,7 +197,7 @@ def ignored? end def pages_job? - name == :pages + name == :pages || name.to_s.start_with?('pages:') end def self.allowed_keys diff --git a/lib/gitlab/ci/config/entry/pages_prefix.rb b/lib/gitlab/ci/config/entry/pages_prefix.rb new file mode 100644 index 00000000000000..5576019b3e6b98 --- /dev/null +++ b/lib/gitlab/ci/config/entry/pages_prefix.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + class Config + module Entry + ## + # Entry that represents the pages prefix + # + class PagesPrefix < ::Gitlab::Config::Entry::Node + include ::Gitlab::Config::Entry::Validatable + + validations do + validates :config, type: String + end + end + end + end + end +end diff --git a/lib/gitlab/ci/yaml_processor/result.rb b/lib/gitlab/ci/yaml_processor/result.rb index 6207b595fc6b05..ad0019ced81340 100644 --- a/lib/gitlab/ci/yaml_processor/result.rb +++ b/lib/gitlab/ci/yaml_processor/result.rb @@ -124,7 +124,8 @@ def build_attributes(name) trigger: job[:trigger], bridge_needs: job.dig(:needs, :bridge)&.first, release: job[:release], - publish: job[:publish] + publish: job[:publish], + pages_prefix: job[:pages_prefix] }.compact }.compact end diff --git a/lib/gitlab/pages/virtual_host_finder.rb b/lib/gitlab/pages/virtual_host_finder.rb index 5fec60188f8513..c47a19113044f2 100644 --- a/lib/gitlab/pages/virtual_host_finder.rb +++ b/lib/gitlab/pages/virtual_host_finder.rb @@ -46,7 +46,7 @@ def by_namespace_domain(name) ::Pages::VirtualDomain.new( trim_prefix: namespace.full_path, - projects: namespace.all_projects_with_pages, + projects: namespace.all_projects_with_pages_deployments, cache: cache ) end -- GitLab