diff --git a/app/controllers/concerns/sessionless_authentication.rb b/app/controllers/concerns/sessionless_authentication.rb index 4304b8565ce8726ec81a94571e30d508956e9c8b..ba06384a37a502eeaae10e392eb9fe117522a6c6 100644 --- a/app/controllers/concerns/sessionless_authentication.rb +++ b/app/controllers/concerns/sessionless_authentication.rb @@ -2,10 +2,10 @@ # == SessionlessAuthentication # -# Controller concern to handle PAT and RSS token authentication methods +# Controller concern to handle PAT, RSS, and static objects token authentication methods # module SessionlessAuthentication - # This filter handles personal access tokens, and atom requests with rss tokens + # This filter handles personal access tokens, atom requests with rss tokens, and static object tokens def authenticate_sessionless_user!(request_format) user = Gitlab::Auth::RequestAuthenticator.new(request).find_sessionless_user(request_format) diff --git a/app/controllers/concerns/static_object_external_storage.rb b/app/controllers/concerns/static_object_external_storage.rb new file mode 100644 index 0000000000000000000000000000000000000000..dbfe0ed3adf0c056ef4150558984808a420d1359 --- /dev/null +++ b/app/controllers/concerns/static_object_external_storage.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module StaticObjectExternalStorage + extend ActiveSupport::Concern + + included do + include ApplicationHelper + end + + def redirect_to_external_storage + return if external_storage_request? + + redirect_to external_storage_url_or_path(request.fullpath, project) + end + + def external_storage_request? + header_token = request.headers['X-Gitlab-External-Storage-Token'] + return false unless header_token.present? + + external_storage_token = Gitlab::CurrentSettings.static_objects_external_storage_auth_token + ActiveSupport::SecurityUtils.secure_compare(header_token, external_storage_token) || + raise(Gitlab::Access::AccessDeniedError) + end +end diff --git a/app/controllers/profiles_controller.rb b/app/controllers/profiles_controller.rb index 1d16ddb1608268dfcb47bb94d9acf5480d27ed6b..958a24b6c0e0a7d90d3771540bc9fca14315dcaf 100644 --- a/app/controllers/profiles_controller.rb +++ b/app/controllers/profiles_controller.rb @@ -46,6 +46,15 @@ def reset_feed_token redirect_to profile_personal_access_tokens_path end + def reset_static_object_token + Users::UpdateService.new(current_user, user: @user).execute! do |user| + user.reset_static_object_token! + end + + redirect_to profile_personal_access_tokens_path, + notice: s_('Profiles|Static object token was successfully reset') + end + # rubocop: disable CodeReuse/ActiveRecord def audit_log @events = AuditEvent.where(entity_type: "User", entity_id: current_user.id) diff --git a/app/controllers/projects/repositories_controller.rb b/app/controllers/projects/repositories_controller.rb index 33dbf934c71def7dcbb32f9b1c236c792f96f424..2ed29b937adcf8972ff1be854d2a468d5ac54fc2 100644 --- a/app/controllers/projects/repositories_controller.rb +++ b/app/controllers/projects/repositories_controller.rb @@ -2,6 +2,9 @@ class Projects::RepositoriesController < Projects::ApplicationController include ExtractsPath + include StaticObjectExternalStorage + + prepend_before_action(only: [:archive]) { authenticate_sessionless_user!(:archive) } # Authorize before_action :require_non_empty_project, except: :create @@ -9,6 +12,7 @@ class Projects::RepositoriesController < Projects::ApplicationController before_action :assign_append_sha, only: :archive before_action :authorize_download_code! before_action :authorize_admin_project!, only: :create + before_action :redirect_to_external_storage, only: :archive, if: :static_objects_external_storage_enabled? def create @project.create_repository diff --git a/app/helpers/application_helper.rb b/app/helpers/application_helper.rb index d9d3cf2bbc538c179cba8506cd431c3fc43123f7..5c2420e80f2b861ed9105bbff69822171b35a935 100644 --- a/app/helpers/application_helper.rb +++ b/app/helpers/application_helper.rb @@ -169,6 +169,25 @@ def support_url Gitlab::CurrentSettings.current_application_settings.help_page_support_url.presence || promo_url + '/getting-help/' end + def static_objects_external_storage_enabled? + Gitlab::CurrentSettings.static_objects_external_storage_enabled? + end + + def external_storage_url_or_path(path, project = @project) + return path unless static_objects_external_storage_enabled? + + uri = URI(Gitlab::CurrentSettings.static_objects_external_storage_url) + path = URI(path) # `path` could have query parameters, so we need to split query and path apart + + query = Rack::Utils.parse_nested_query(path.query) + query['token'] = current_user.static_object_token unless project.public? + + uri.path = path.path + uri.query = query.to_query unless query.empty? + + uri.to_s + end + def page_filter_path(options = {}) without = options.delete(:without) diff --git a/app/helpers/application_settings_helper.rb b/app/helpers/application_settings_helper.rb index 10243e0f7eead272633845b67bf8527bb268f466..9a1a2b3a79c0f376440f9769bbf58fe992258b53 100644 --- a/app/helpers/application_settings_helper.rb +++ b/app/helpers/application_settings_helper.rb @@ -168,6 +168,8 @@ def visible_attributes :asset_proxy_secret_key, :asset_proxy_url, :asset_proxy_whitelist, + :static_objects_external_storage_auth_token, + :static_objects_external_storage_url, :authorized_keys_enabled, :auto_devops_enabled, :auto_devops_domain, diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index c8f84d53fc583f115179ca9a0b3224c0bcea0913..3bb29213d8f75a3f2cda672c5df7a2ca5d94fbfc 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -8,6 +8,7 @@ class ApplicationSetting < ApplicationRecord add_authentication_token_field :runners_registration_token, encrypted: -> { Feature.enabled?(:application_settings_tokens_optional_encryption, default_enabled: true) ? :optional : :required } add_authentication_token_field :health_check_access_token + add_authentication_token_field :static_objects_external_storage_auth_token belongs_to :instance_administration_project, class_name: "Project" @@ -211,6 +212,13 @@ class ApplicationSetting < ApplicationRecord allow_blank: false, if: :asset_proxy_enabled? + validates :static_objects_external_storage_url, + addressable_url: true, allow_blank: true + + validates :static_objects_external_storage_auth_token, + presence: true, + if: :static_objects_external_storage_url? + SUPPORTED_KEY_TYPES.each do |type| validates :"#{type}_key_restriction", presence: true, key_restriction: { type: type } end diff --git a/app/models/application_setting_implementation.rb b/app/models/application_setting_implementation.rb index f402c0e2775029fe7b9397f7a2dd72c60774c970..8d9597aa5a494f98cf1e03bdcd87cf2cce185c1e 100644 --- a/app/models/application_setting_implementation.rb +++ b/app/models/application_setting_implementation.rb @@ -306,6 +306,10 @@ def archive_builds_older_than archive_builds_in_seconds.seconds.ago if archive_builds_in_seconds end + def static_objects_external_storage_enabled? + static_objects_external_storage_url.present? + end + private def array_to_string(arr) diff --git a/app/models/user.rb b/app/models/user.rb index 9b01ba0b682d88b2b80ee1958ae8c59b04cd9446..db447b80da87653b5b4457d2b58e32583f330dea 100644 --- a/app/models/user.rb +++ b/app/models/user.rb @@ -31,6 +31,7 @@ class User < ApplicationRecord add_authentication_token_field :incoming_email_token, token_generator: -> { SecureRandom.hex.to_i(16).to_s(36) } add_authentication_token_field :feed_token + add_authentication_token_field :static_object_token default_value_for :admin, false default_value_for(:external) { Gitlab::CurrentSettings.user_default_external } @@ -1437,6 +1438,13 @@ def feed_token ensure_feed_token! end + # Each existing user needs to have a `static_object_token`. + # We do this on read since migrating all existing users is not a feasible + # solution. + def static_object_token + ensure_static_object_token! + end + def sync_attribute?(attribute) return true if ldap_user? && attribute == :email diff --git a/app/views/admin/application_settings/_repository_static_objects.html.haml b/app/views/admin/application_settings/_repository_static_objects.html.haml new file mode 100644 index 0000000000000000000000000000000000000000..03aa48b2282ded7082806ff383fad6caab101186 --- /dev/null +++ b/app/views/admin/application_settings/_repository_static_objects.html.haml @@ -0,0 +1,18 @@ += form_for @application_setting, url: repository_admin_application_settings_path(anchor: 'js-repository-static-objects-settings'), html: { class: 'fieldset-form' } do |f| + = form_errors(@application_setting) + + %fieldset + .form-group + = f.label :static_objects_external_storage_url, class: 'label-bold' do + = _('External storage URL') + = f.text_field :static_objects_external_storage_url, class: 'form-control' + %span.form-text.text-muted#static_objects_external_storage_url_help_block + = _('URL of the external storage that will serve the repository static objects (e.g. archives, blobs, ...).') + .form-group + = f.label :static_objects_external_storage_auth_token, class: 'label-bold' do + = _('External storage authentication token') + = f.text_field :static_objects_external_storage_auth_token, class: 'form-control' + %span.form-text.text-muted#static_objects_external_storage_auth_token_help_block + = _('A secure token that identifies an external storage request.') + + = f.submit _('Save changes'), class: "btn btn-success" diff --git a/app/views/admin/application_settings/repository.html.haml b/app/views/admin/application_settings/repository.html.haml index b50a0dd5a183f0125a98504a0e8c17676ed5bc69..25f8b6541b5a522e81ce2c69e03c2e87b2fd59e4 100644 --- a/app/views/admin/application_settings/repository.html.haml +++ b/app/views/admin/application_settings/repository.html.haml @@ -34,3 +34,14 @@ = _('Configure automatic git checks and housekeeping on repositories.') .settings-content = render 'repository_check' + +%section.settings.as-repository-static-objects.no-animate#js-repository-static-objects-settings{ class: ('expanded' if expanded_by_default?) } + .settings-header + %h4 + = _('Repository static objects') + %button.btn.btn-default.js-settings-toggle{ type: 'button' } + = expanded_by_default? ? _('Collapse') : _('Expand') + %p + = _('Serve repository static objects (e.g. archives, blobs, ...) from an external storage (e.g. a CDN).') + .settings-content + = render 'repository_static_objects' diff --git a/app/views/profiles/personal_access_tokens/index.html.haml b/app/views/profiles/personal_access_tokens/index.html.haml index 08a39fc4f58193615eccbc8f965b9657bdd761aa..d9e94908b802efaaed6d402c5b7c15f520341f6e 100644 --- a/app/views/profiles/personal_access_tokens/index.html.haml +++ b/app/views/profiles/personal_access_tokens/index.html.haml @@ -54,3 +54,23 @@ - reset_link = link_to s_('AccessTokens|reset it'), [:reset, :incoming_email_token, :profile], method: :put, data: { confirm: s_('AccessTokens|Are you sure? Any issue email addresses currently in use will stop working.') } - reset_message = s_('AccessTokens|Keep this token secret. Anyone who gets ahold of it can create issues as if they were you. You should %{link_reset_it} if that ever happens.') % { link_reset_it: reset_link } = reset_message.html_safe + +- if static_objects_external_storage_enabled? + %hr + .row.prepend-top-default + .col-lg-4 + %h4.prepend-top-0 + = s_('AccessTokens|Static object token') + %p + = s_('AccessTokens|Your static object token is used to authenticate you when repository static objects (e.g. archives, blobs, ...) are being served from an external storage.') + %p + = s_('AccessTokens|It cannot be used to access any other data.') + .col-lg-8 + = label_tag :static_object_token, s_('AccessTokens|Static object token'), class: "label-bold" + = text_field_tag :static_object_token, current_user.static_object_token, class: 'form-control', readonly: true, onclick: 'this.select()' + %p.form-text.text-muted + - reset_link = url_for [:reset, :static_object_token, :profile] + - reset_link_start = ''.html_safe % { confirm: s_('AccessTokens|Are you sure?'), url: reset_link } + - reset_link_end = ''.html_safe + - reset_message = s_('AccessTokens|Keep this token secret. Anyone who gets ahold of it can access repository static objects as if they were you. You should %{reset_link_start}reset it%{reset_link_end} if that ever happens.') % { reset_link_start: reset_link_start, reset_link_end: reset_link_end } + = reset_message.html_safe diff --git a/app/views/projects/buttons/_download_links.html.haml b/app/views/projects/buttons/_download_links.html.haml index d344167a6c54fea85e8201b9342b0731ec599927..b256d94065bc485feb2d1cfab4219e7be114563f 100644 --- a/app/views/projects/buttons/_download_links.html.haml +++ b/app/views/projects/buttons/_download_links.html.haml @@ -2,4 +2,5 @@ .btn-group.ml-0.w-100 - formats.each do |(fmt, extra_class)| - = link_to fmt, project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: fmt), rel: 'nofollow', download: '', class: "btn btn-xs #{extra_class}" + - archive_path = project_archive_path(project, id: tree_join(ref, archive_prefix), path: path, format: fmt) + = link_to fmt, external_storage_url_or_path(archive_path), rel: 'nofollow', download: '', class: "btn btn-xs #{extra_class}" diff --git a/changelogs/unreleased/static-objects-external-storage.yml b/changelogs/unreleased/static-objects-external-storage.yml new file mode 100644 index 0000000000000000000000000000000000000000..fd687b2262ce2d7b85b871aeae4c96c5603d86f4 --- /dev/null +++ b/changelogs/unreleased/static-objects-external-storage.yml @@ -0,0 +1,5 @@ +--- +title: Enable serving static objects from an external storage +merge_request: 31025 +author: +type: added diff --git a/config/routes/profile.rb b/config/routes/profile.rb index 83a2b33514b5ece04a7f902b2d849995246bb2f9..403f430850e1ae4f5daf8b6074fdb0d259d85084 100644 --- a/config/routes/profile.rb +++ b/config/routes/profile.rb @@ -8,6 +8,7 @@ put :reset_incoming_email_token put :reset_feed_token + put :reset_static_object_token put :update_username end diff --git a/db/migrate/20190722104947_add_static_object_token_to_users.rb b/db/migrate/20190722104947_add_static_object_token_to_users.rb new file mode 100644 index 0000000000000000000000000000000000000000..6ef85d9acaa2f7fbedba92588b6b5fe399d39279 --- /dev/null +++ b/db/migrate/20190722104947_add_static_object_token_to_users.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddStaticObjectTokenToUsers < ActiveRecord::Migration[5.2] + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_column :users, :static_object_token, :string, limit: 255 + end + + def down + remove_column :users, :static_object_token + end +end diff --git a/db/migrate/20190722132830_add_static_objects_external_storage_columns_to_application_settings.rb b/db/migrate/20190722132830_add_static_objects_external_storage_columns_to_application_settings.rb new file mode 100644 index 0000000000000000000000000000000000000000..a23e6ed66cdb32a1c81fafb841b20af18ab2b6d0 --- /dev/null +++ b/db/migrate/20190722132830_add_static_objects_external_storage_columns_to_application_settings.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddStaticObjectsExternalStorageColumnsToApplicationSettings < ActiveRecord::Migration[5.2] + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def change + add_column :application_settings, :static_objects_external_storage_url, :string, limit: 255 + add_column :application_settings, :static_objects_external_storage_auth_token, :string, limit: 255 + end +end diff --git a/db/migrate/20190725183432_add_index_to_index_on_static_object_token.rb b/db/migrate/20190725183432_add_index_to_index_on_static_object_token.rb new file mode 100644 index 0000000000000000000000000000000000000000..423c45b954382ee2681785549f8ee81817a3f66e --- /dev/null +++ b/db/migrate/20190725183432_add_index_to_index_on_static_object_token.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class AddIndexToIndexOnStaticObjectToken < ActiveRecord::Migration[5.2] + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + disable_ddl_transaction! + + def up + add_concurrent_index :users, :static_object_token, unique: true + end + + def down + remove_concurrent_index :users, :static_object_token + end +end diff --git a/db/schema.rb b/db/schema.rb index 1157bd94cbbdb76db98cce2082e75c0baa0ab9bd..154fefe0d4b4aa54acfb46ab8e9a1045f70cf7a0 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -284,6 +284,8 @@ t.text "asset_proxy_whitelist" t.text "encrypted_asset_proxy_secret_key" t.string "encrypted_asset_proxy_secret_key_iv" + t.string "static_objects_external_storage_url", limit: 255 + t.string "static_objects_external_storage_auth_token", limit: 255 t.index ["custom_project_templates_group_id"], name: "index_application_settings_on_custom_project_templates_group_id" t.index ["file_template_project_id"], name: "index_application_settings_on_file_template_project_id" t.index ["instance_administration_project_id"], name: "index_applicationsettings_on_instance_administration_project_id" @@ -3565,6 +3567,7 @@ t.integer "bot_type", limit: 2 t.string "first_name", limit: 255 t.string "last_name", limit: 255 + t.string "static_object_token", limit: 255 t.index ["accepted_term_id"], name: "index_users_on_accepted_term_id" t.index ["admin"], name: "index_users_on_admin" t.index ["bot_type"], name: "index_users_on_bot_type" @@ -3584,6 +3587,7 @@ t.index ["state"], name: "index_users_on_state" t.index ["state"], name: "index_users_on_state_and_internal", where: "(ghost IS NOT TRUE)" t.index ["state"], name: "index_users_on_state_and_internal_ee", where: "((ghost IS NOT TRUE) AND (bot_type IS NULL))" + t.index ["static_object_token"], name: "index_users_on_static_object_token", unique: true t.index ["unconfirmed_email"], name: "index_users_on_unconfirmed_email", where: "(unconfirmed_email IS NOT NULL)" t.index ["username"], name: "index_users_on_username" t.index ["username"], name: "index_users_on_username_trigram", opclass: :gin_trgm_ops, using: :gin diff --git a/doc/administration/index.md b/doc/administration/index.md index b58291b74785b7c4133c9f555bd5fc93af428b5e..df3501ae95088dd04ca8207d2948f2dee6329d56 100644 --- a/doc/administration/index.md +++ b/doc/administration/index.md @@ -143,6 +143,7 @@ Learn how to install, configure, update, and maintain your GitLab instance. - [Repository storage types](repository_storage_types.md): Information about the different repository storage types. - [Repository storage rake tasks](raketasks/storage.md): A collection of rake tasks to list and migrate existing projects and attachments associated with it from Legacy storage to Hashed storage. - [Limit repository size](../user/admin_area/settings/account_and_limit_settings.md): Set a hard limit for your repositories' size. **(STARTER ONLY)** +- [Static objects external storage](static_objects_external_storage.md): Set external storage for static objects in a repository. ## Continuous Integration settings diff --git a/doc/administration/static_objects_external_storage.md b/doc/administration/static_objects_external_storage.md new file mode 100644 index 0000000000000000000000000000000000000000..e4d60c771991db45d555e22decb981577b3c9e63 --- /dev/null +++ b/doc/administration/static_objects_external_storage.md @@ -0,0 +1,50 @@ +# Static objects external storage + +> [Introduced](https://gitlab.com/gitlab-org/gitlab-ce/merge_requests/31025) in GitLab 12.3. + +GitLab can be configured to serve repository static objects (for example, archives) from an external +storage, such as a CDN. + +## Configuring + +To configure external storage for static objects: + +1. Navigate to **Admin Area > Settings > Repository**. +1. Expand the **Repository static objects** section. +1. Enter the base URL and an arbitrary token. + +The token is required to distinguish requests coming from the external storage, so users don't +circumvent the external storage and go for the application directly. The token is expected to be +set in the `X-Gitlab-External-Storage-Token` header in requests originating from the external +storage. + +## Serving private static objects + +GitLab will append a user-specific token for static object URLs that belong to private projects, +so an external storage can be authenticated on behalf of the user. When processing requests originating +from the external storage, GitLab will look for the token in the `token` query parameter or in +the `X-Gitlab-Static-Object-Token` header to check the user's ability to access the requested object. + +## Requests flow example + +The following example shows a sequence of requests and responses between the user, +GitLab, and the CDN: + +```mermaid +sequenceDiagram + User->>GitLab: GET /project/-/archive/master.zip + GitLab->>User: 302 Found + Note over User,GitLab: Location: https://cdn.com/project/-/archive/master.zip?token=secure-user-token + User->>CDN: GET /project/-/archive/master.zip?token=secure-user-token + alt object not in cache + CDN->>GitLab: GET /project/-/archive/master.zip + Note over CDN,GitLab: X-Gitlab-External-Storage-Token: secure-cdn-token
X-Gitlab-Static-Object-Token: secure-user-token + GitLab->>CDN: 200 OK + CDN->>User: master.zip + else object in cache + CDN->>GitLab: GET /project/-/archive/master.zip + Note over CDN,GitLab: X-Gitlab-External-Storage-Token: secure-cdn-token
X-Gitlab-Static-Object-Token: secure-user-token
If-None-Match: etag-value + GitLab->>CDN: 304 Not Modified + CDN->>User: master.zip + end +``` diff --git a/lib/gitlab/auth/request_authenticator.rb b/lib/gitlab/auth/request_authenticator.rb index 176766d1a8b95bf7e01346810fceca68ee6109f1..aca8804b04c616506821440da8264c4f889f8251 100644 --- a/lib/gitlab/auth/request_authenticator.rb +++ b/lib/gitlab/auth/request_authenticator.rb @@ -24,7 +24,9 @@ def user(request_formats) end def find_sessionless_user(request_format) - find_user_from_web_access_token(request_format) || find_user_from_feed_token(request_format) + find_user_from_web_access_token(request_format) || + find_user_from_feed_token(request_format) || + find_user_from_static_object_token(request_format) rescue Gitlab::Auth::AuthenticationError nil end diff --git a/lib/gitlab/auth/user_auth_finders.rb b/lib/gitlab/auth/user_auth_finders.rb index 0154cd4fbace91f37f54b40455034f8f8cb558ae..e2f562c084377b88c1f93ddc71cdb4b942ef0f09 100644 --- a/lib/gitlab/auth/user_auth_finders.rb +++ b/lib/gitlab/auth/user_auth_finders.rb @@ -30,6 +30,15 @@ def find_user_from_warden current_request.env['warden']&.authenticate if verified_request? end + def find_user_from_static_object_token(request_format) + return unless valid_static_objects_format?(request_format) + + token = current_request.params[:token].presence || current_request.headers['X-Gitlab-Static-Object-Token'].presence + return unless token + + User.find_by_static_object_token(token) || raise(UnauthorizedError) + end + def find_user_from_feed_token(request_format) return unless valid_rss_format?(request_format) @@ -156,6 +165,15 @@ def valid_rss_format?(request_format) end end + def valid_static_objects_format?(request_format) + case request_format + when :archive + archive_request? + else + false + end + end + def rss_request? current_request.path.ends_with?('.atom') || current_request.format.atom? end @@ -167,6 +185,10 @@ def ics_request? def api_request? current_request.path.starts_with?("/api/") end + + def archive_request? + current_request.path.include?('/-/archive/') + end end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 8ff14593bb867d2f5ff982c4834e009fd1984169..353d370d14f635390b9a2c7dfd2af0d3d0435e66 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -651,6 +651,9 @@ msgstr "" msgid "A regular expression that will be used to find the test coverage output in the job trace. Leave blank to disable" msgstr "" +msgid "A secure token that identifies an external storage request." +msgstr "" + msgid "A user with write access to the source branch selected this option" msgstr "" @@ -720,6 +723,9 @@ msgstr "" msgid "AccessTokens|Access Tokens" msgstr "" +msgid "AccessTokens|Are you sure?" +msgstr "" + msgid "AccessTokens|Are you sure? Any RSS or calendar URLs currently in use will stop working." msgstr "" @@ -738,6 +744,9 @@ msgstr "" msgid "AccessTokens|It cannot be used to access any other data." msgstr "" +msgid "AccessTokens|Keep this token secret. Anyone who gets ahold of it can access repository static objects as if they were you. You should %{reset_link_start}reset it%{reset_link_end} if that ever happens." +msgstr "" + msgid "AccessTokens|Keep this token secret. Anyone who gets ahold of it can create issues as if they were you. You should %{link_reset_it} if that ever happens." msgstr "" @@ -747,6 +756,9 @@ msgstr "" msgid "AccessTokens|Personal Access Tokens" msgstr "" +msgid "AccessTokens|Static object token" +msgstr "" + msgid "AccessTokens|They are the only accepted password when you have Two-Factor Authentication (2FA) enabled." msgstr "" @@ -762,6 +774,9 @@ msgstr "" msgid "AccessTokens|Your incoming email token is used to authenticate you when you create a new issue by email, and is included in your personal project-specific email addresses." msgstr "" +msgid "AccessTokens|Your static object token is used to authenticate you when repository static objects (e.g. archives, blobs, ...) are being served from an external storage." +msgstr "" + msgid "AccessTokens|reset it" msgstr "" @@ -6205,6 +6220,12 @@ msgstr "" msgid "External authorization request timeout" msgstr "" +msgid "External storage URL" +msgstr "" + +msgid "External storage authentication token" +msgstr "" + msgid "ExternalAuthorizationService|Classification label" msgstr "" @@ -11581,6 +11602,9 @@ msgstr "" msgid "Profiles|Some options are unavailable for LDAP accounts" msgstr "" +msgid "Profiles|Static object token was successfully reset" +msgstr "" + msgid "Profiles|Tell us about yourself in fewer than 250 characters" msgstr "" @@ -12912,6 +12936,9 @@ msgstr "" msgid "Repository mirror" msgstr "" +msgid "Repository static objects" +msgstr "" + msgid "Repository storage" msgstr "" @@ -13707,6 +13734,9 @@ msgstr "" msgid "SeriesFinalConjunction|and" msgstr "" +msgid "Serve repository static objects (e.g. archives, blobs, ...) from an external storage (e.g. a CDN)." +msgstr "" + msgid "Server supports batch API only, please update your Git LFS client to version 1.0.1 and up." msgstr "" @@ -16373,6 +16403,9 @@ msgstr "" msgid "URL" msgstr "" +msgid "URL of the external storage that will serve the repository static objects (e.g. archives, blobs, ...)." +msgstr "" + msgid "Unable to apply suggestions to a deleted line." msgstr "" diff --git a/spec/controllers/concerns/static_object_external_storage_spec.rb b/spec/controllers/concerns/static_object_external_storage_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..3a0219ddaa1fa419cb3211d98ef5fbd6c07d561c --- /dev/null +++ b/spec/controllers/concerns/static_object_external_storage_spec.rb @@ -0,0 +1,96 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe StaticObjectExternalStorage do + controller(Projects::ApplicationController) do + include StaticObjectExternalStorage # rubocop:disable RSpec/DescribedClass + + before_action :redirect_to_external_storage, if: :static_objects_external_storage_enabled? + + def show + head :ok + end + end + + let(:project) { create(:project, :public) } + let(:user) { create(:user, static_object_token: 'hunter1') } + + before do + project.add_developer(user) + sign_in(user) + end + + context 'when external storage is not configured' do + it 'calls the action normally' do + expect(Gitlab::CurrentSettings.static_objects_external_storage_url).to be_blank + + do_request + + expect(response).to have_gitlab_http_status(200) + end + end + + context 'when external storage is configured' do + before do + allow_any_instance_of(ApplicationSetting).to receive(:static_objects_external_storage_url).and_return('https://cdn.gitlab.com') + allow_any_instance_of(ApplicationSetting).to receive(:static_objects_external_storage_auth_token).and_return('letmein') + + routes.draw { get '/:namespace_id/:id' => 'projects/application#show' } + end + + context 'when external storage token is empty' do + let(:base_redirect_url) { "https://cdn.gitlab.com/#{project.namespace.to_param}/#{project.to_param}" } + + context 'when project is public' do + it 'redirects to external storage URL without adding a token parameter' do + do_request + + expect(response).to redirect_to(base_redirect_url) + end + end + + context 'when project is not public' do + let(:project) { create(:project, :private) } + + it 'redirects to external storage URL a token parameter added' do + do_request + + expect(response).to redirect_to("#{base_redirect_url}?token=#{user.static_object_token}") + end + + context 'when path includes extra parameters' do + it 'includes the parameters in the redirect URL' do + do_request(foo: 'bar') + + expect(response.location).to eq("#{base_redirect_url}?foo=bar&token=#{user.static_object_token}") + end + end + end + end + + context 'when external storage token is present' do + context 'when token is correct' do + it 'calls the action normally' do + request.headers['X-Gitlab-External-Storage-Token'] = 'letmein' + do_request + + expect(response).to have_gitlab_http_status(200) + end + end + + context 'when token is incorrect' do + it 'return 403' do + request.headers['X-Gitlab-External-Storage-Token'] = 'donotletmein' + do_request + + expect(response).to have_gitlab_http_status(403) + end + end + end + end + + def do_request(extra_params = {}) + get :show, params: { namespace_id: project.namespace, id: project }.merge(extra_params) + end +end diff --git a/spec/controllers/projects/repositories_controller_spec.rb b/spec/controllers/projects/repositories_controller_spec.rb index fcab4d73dca30cef1f40eda94de9db0c81c2cb41..084644484c55354af00a7c7fc99935567cc02171 100644 --- a/spec/controllers/projects/repositories_controller_spec.rb +++ b/spec/controllers/projects/repositories_controller_spec.rb @@ -125,5 +125,59 @@ def get_archive(id = 'feature') end end end + + context 'as a sessionless user' do + let(:user) { create(:user) } + + before do + project.add_developer(user) + end + + context 'when no token is provided' do + it 'redirects to sign in page' do + get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master' }, format: 'zip' + + expect(response).to have_gitlab_http_status(302) + end + end + + context 'when a token param is present' do + context 'when token is correct' do + it 'calls the action normally' do + get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master', token: user.static_object_token }, format: 'zip' + + expect(response).to have_gitlab_http_status(200) + end + end + + context 'when token is incorrect' do + it 'redirects to sign in page' do + get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master', token: 'foobar' }, format: 'zip' + + expect(response).to have_gitlab_http_status(302) + end + end + end + + context 'when a token header is present' do + context 'when token is correct' do + it 'calls the action normally' do + request.headers['X-Gitlab-Static-Object-Token'] = user.static_object_token + get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master' }, format: 'zip' + + expect(response).to have_gitlab_http_status(200) + end + end + + context 'when token is incorrect' do + it 'redirects to sign in page' do + request.headers['X-Gitlab-Static-Object-Token'] = 'foobar' + get :archive, params: { namespace_id: project.namespace, project_id: project, id: 'master' }, format: 'zip' + + expect(response).to have_gitlab_http_status(302) + end + end + end + end end end diff --git a/spec/features/projects/branches/download_buttons_spec.rb b/spec/features/projects/branches/download_buttons_spec.rb index 401425187b0da3fb44cd74e9faae61602f7298db..e0b0e22823e3dafbbb8f710be0ae6a9201325023 100644 --- a/spec/features/projects/branches/download_buttons_spec.rb +++ b/spec/features/projects/branches/download_buttons_spec.rb @@ -29,6 +29,11 @@ end describe 'when checking branches' do + it_behaves_like 'archive download buttons' do + let(:ref) { 'binary-encoding' } + let(:path_to_visit) { project_branches_filtered_path(project, state: 'all', search: ref) } + end + context 'with artifacts' do before do visit project_branches_filtered_path(project, state: 'all', search: 'binary-encoding') diff --git a/spec/features/projects/files/download_buttons_spec.rb b/spec/features/projects/files/download_buttons_spec.rb index a4889f8d4c423bd011c61c8fed4f43b981384276..871f5212dddab7ec7de573e64750b718bab24631 100644 --- a/spec/features/projects/files/download_buttons_spec.rb +++ b/spec/features/projects/files/download_buttons_spec.rb @@ -24,11 +24,17 @@ before do sign_in(user) project.add_developer(user) + end - visit project_tree_path(project, project.default_branch) + it_behaves_like 'archive download buttons' do + let(:path_to_visit) { project_tree_path(project, project.default_branch) } end context 'with artifacts' do + before do + visit project_tree_path(project, project.default_branch) + end + it 'shows download artifacts button' do href = latest_succeeded_project_artifacts_path(project, "#{project.default_branch}/download", job: 'build') diff --git a/spec/features/projects/show/download_buttons_spec.rb b/spec/features/projects/show/download_buttons_spec.rb index 5e7453bcdb708a2ced3f04e93171f88581394fbb..0d60906942604c05be3087ecd133cf851bd321c3 100644 --- a/spec/features/projects/show/download_buttons_spec.rb +++ b/spec/features/projects/show/download_buttons_spec.rb @@ -29,6 +29,8 @@ end describe 'when checking project main page' do + it_behaves_like 'archive download buttons' + context 'with artifacts' do before do visit project_path(project) diff --git a/spec/features/projects/tags/download_buttons_spec.rb b/spec/features/projects/tags/download_buttons_spec.rb index 76b2704ae494f33c1ffdf5e204d19475eb2c7416..64141cf5dc91444e7e707945aa71ad98a6d23b36 100644 --- a/spec/features/projects/tags/download_buttons_spec.rb +++ b/spec/features/projects/tags/download_buttons_spec.rb @@ -30,6 +30,11 @@ end describe 'when checking tags' do + it_behaves_like 'archive download buttons' do + let(:path_to_visit) { project_tags_path(project) } + let(:ref) { tag } + end + context 'with artifacts' do before do visit project_tags_path(project) diff --git a/spec/helpers/application_helper_spec.rb b/spec/helpers/application_helper_spec.rb index b81249a1e298274a0e7a442e4da6590a71621ce8..4a3ff7e009545d3193fee36d9ca9c8729770222e 100644 --- a/spec/helpers/application_helper_spec.rb +++ b/spec/helpers/application_helper_spec.rb @@ -195,4 +195,41 @@ def element(*arguments) end end end + + describe '#external_storage_url_or_path' do + let(:project) { create(:project) } + + context 'when external storage is disabled' do + it 'returns the passed path' do + expect(helper.external_storage_url_or_path('/foo/bar', project)).to eq('/foo/bar') + end + end + + context 'when external storage is enabled' do + let(:user) { create(:user, static_object_token: 'hunter1') } + + before do + allow_any_instance_of(ApplicationSetting).to receive(:static_objects_external_storage_url).and_return('https://cdn.gitlab.com') + allow(helper).to receive(:current_user).and_return(user) + end + + it 'returns the external storage URL prepended to the path' do + expect(helper.external_storage_url_or_path('/foo/bar', project)).to eq("https://cdn.gitlab.com/foo/bar?token=#{user.static_object_token}") + end + + it 'preserves the path query parameters' do + url = helper.external_storage_url_or_path('/foo/bar?unicode=1', project) + + expect(url).to eq("https://cdn.gitlab.com/foo/bar?token=#{user.static_object_token}&unicode=1") + end + + context 'when project is public' do + let(:project) { create(:project, :public) } + + it 'returns does not append a token parameter' do + expect(helper.external_storage_url_or_path('/foo/bar', project)).to eq('https://cdn.gitlab.com/foo/bar') + end + end + end + end end diff --git a/spec/lib/gitlab/auth/user_auth_finders_spec.rb b/spec/lib/gitlab/auth/user_auth_finders_spec.rb index 41265da97a43a2b09d139841b1b3b0e87f36a7c2..dd8070c124082cf617ebc391e3e5ab5bce86290e 100644 --- a/spec/lib/gitlab/auth/user_auth_finders_spec.rb +++ b/spec/lib/gitlab/auth/user_auth_finders_spec.rb @@ -115,6 +115,60 @@ def set_param(key, value) end end + describe '#find_user_from_static_object_token' do + context 'when request format is archive' do + before do + env['SCRIPT_NAME'] = 'project/-/archive/master.zip' + end + + context 'when token header param is present' do + context 'when token is correct' do + it 'returns the user' do + request.headers['X-Gitlab-Static-Object-Token'] = user.static_object_token + + expect(find_user_from_static_object_token(:archive)).to eq(user) + end + end + + context 'when token is incorrect' do + it 'returns the user' do + request.headers['X-Gitlab-Static-Object-Token'] = 'foobar' + + expect { find_user_from_static_object_token(:archive) }.to raise_error(Gitlab::Auth::UnauthorizedError) + end + end + end + + context 'when token query param is present' do + context 'when token is correct' do + it 'returns the user' do + set_param(:token, user.static_object_token) + + expect(find_user_from_static_object_token(:archive)).to eq(user) + end + end + + context 'when token is incorrect' do + it 'returns the user' do + set_param(:token, 'foobar') + + expect { find_user_from_static_object_token(:archive) }.to raise_error(Gitlab::Auth::UnauthorizedError) + end + end + end + end + + context 'when request format is not archive' do + before do + env['script_name'] = 'url' + end + + it 'returns nil' do + expect(find_user_from_static_object_token(:foo)).to be_nil + end + end + end + describe '#find_user_from_access_token' do let(:personal_access_token) { create(:personal_access_token, user: user) } diff --git a/spec/models/application_setting_spec.rb b/spec/models/application_setting_spec.rb index 4f7a6d102b8bb6555454f64b12c9b7c1c3009792..d12f9b9100a24ab5d222e605be603517ba26449a 100644 --- a/spec/models/application_setting_spec.rb +++ b/spec/models/application_setting_spec.rb @@ -48,6 +48,10 @@ it { is_expected.not_to allow_value(nil).for(:outbound_local_requests_whitelist) } it { is_expected.to allow_value([]).for(:outbound_local_requests_whitelist) } + it { is_expected.to allow_value(nil).for(:static_objects_external_storage_url) } + it { is_expected.to allow_value(http).for(:static_objects_external_storage_url) } + it { is_expected.to allow_value(https).for(:static_objects_external_storage_url) } + context "when user accepted let's encrypt terms of service" do before do setting.update(lets_encrypt_terms_of_service_accepted: true) @@ -420,6 +424,16 @@ def expect_invalid end end end + + context 'static objects external storage' do + context 'when URL is set' do + before do + subject.static_objects_external_storage_url = http + end + + it { is_expected.not_to allow_value(nil).for(:static_objects_external_storage_auth_token) } + end + end end context 'restrict creating duplicates' do diff --git a/spec/models/user_spec.rb b/spec/models/user_spec.rb index 6722a3c627d0891a48338c75591a7d94ae9099db..c339fad778b0c0070b4397b6de4aaf0bafcca1c8 100644 --- a/spec/models/user_spec.rb +++ b/spec/models/user_spec.rb @@ -945,6 +945,16 @@ end end + describe 'static object token' do + it 'ensures a static object token on read' do + user = create(:user, static_object_token: nil) + static_object_token = user.static_object_token + + expect(static_object_token).not_to be_blank + expect(user.reload.static_object_token).to eq static_object_token + end + end + describe '#recently_sent_password_reset?' do it 'is false when reset_password_sent_at is nil' do user = build_stubbed(:user, reset_password_sent_at: nil) diff --git a/spec/support/shared_examples/features/archive_download_buttons_shared_examples.rb b/spec/support/shared_examples/features/archive_download_buttons_shared_examples.rb new file mode 100644 index 0000000000000000000000000000000000000000..920fcbde4839f369b859329a9629652333cf3958 --- /dev/null +++ b/spec/support/shared_examples/features/archive_download_buttons_shared_examples.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +shared_examples 'archive download buttons' do + let(:formats) { %w(zip tar.gz tar.bz2 tar) } + let(:path_to_visit) { project_path(project) } + let(:ref) { project.default_branch } + + context 'when static objects external storage is enabled' do + before do + allow_any_instance_of(ApplicationSetting).to receive(:static_objects_external_storage_url).and_return('https://cdn.gitlab.com') + visit path_to_visit + end + + context 'private project' do + it 'shows archive download buttons with external storage URL prepended and user token appended to their href' do + formats.each do |format| + path = archive_path(project, ref, format) + uri = URI('https://cdn.gitlab.com') + uri.path = path + uri.query = "token=#{user.static_object_token}" + + expect(page).to have_link format, href: uri.to_s + end + end + end + + context 'public project' do + let(:project) { create(:project, :repository, :public) } + + it 'shows archive download buttons with external storage URL prepended to their href' do + formats.each do |format| + path = archive_path(project, ref, format) + uri = URI('https://cdn.gitlab.com') + uri.path = path + + expect(page).to have_link format, href: uri.to_s + end + end + end + end + + context 'when static objects external storage is disabled' do + before do + visit path_to_visit + end + + it 'shows default archive download buttons' do + formats.each do |format| + path = archive_path(project, ref, format) + + expect(page).to have_link format, href: path + end + end + end + + def archive_path(project, ref, format) + project_archive_path(project, id: "#{ref}/#{project.path}-#{ref}", path: nil, format: format) + end +end