From df74cf0a74cc6fa0e2ce239cf57c748d75ca44b8 Mon Sep 17 00:00:00 2001 From: Vasilii Iakliushin Date: Sat, 11 Nov 2023 18:20:21 +0100 Subject: [PATCH] Add UI to support SHA256 repositories creation Contributes to https://gitlab.com/gitlab-org/gitlab/-/issues/426803 **Problem** Gitaly provides a support for SHA256 repositories. But this feature is not supported by Rails application. **Solution** Add UI to allow users to create new projects with SHA256 repositories. Changelog: added --- app/controllers/projects_controller.rb | 1 + app/models/project.rb | 4 +- app/models/repository.rb | 3 ++ app/services/projects/create_service.rb | 7 +++- .../projects/_new_project_fields.html.haml | 9 +++++ .../support_sha256_repositories.yml | 8 ++++ lib/gitlab/git/repository.rb | 4 +- .../gitaly_client/repository_service.rb | 13 ++++++- locale/gitlab.pot | 6 +++ spec/factories/projects.rb | 11 ++++-- .../projects/user_creates_project_spec.rb | 39 +++++++++++++++++++ .../gitaly_client/repository_service_spec.rb | 34 ++++++++++++++++ spec/models/project_spec.rb | 9 ++++- spec/services/projects/create_service_spec.rb | 25 ++++++++++++ 14 files changed, 162 insertions(+), 11 deletions(-) create mode 100644 config/feature_flags/development/support_sha256_repositories.yml diff --git a/app/controllers/projects_controller.rb b/app/controllers/projects_controller.rb index 6f15bc553bfd4c..93cea43eed2f6a 100644 --- a/app/controllers/projects_controller.rb +++ b/app/controllers/projects_controller.rb @@ -510,6 +510,7 @@ def project_params_attributes :merge_method, :initialize_with_sast, :initialize_with_readme, + :use_sha256_repository, :ci_separated_caches, :suggestion_commit_message, :packages_enabled, diff --git a/app/models/project.rb b/app/models/project.rb index bc6d2600d832e5..ae19e835928e61 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1938,11 +1938,11 @@ def track_project_repository cleanup if replicate_object_pool_on_move_ff_enabled? end - def create_repository(force: false, default_branch: nil) + def create_repository(force: false, default_branch: nil, object_format: nil) # Forked import is handled asynchronously return if forked? && !force - repository.create_repository(default_branch) + repository.create_repository(default_branch, object_format: object_format) repository.after_create true diff --git a/app/models/repository.rb b/app/models/repository.rb index e639a389e0a58c..f035829a8b0d94 100644 --- a/app/models/repository.rb +++ b/app/models/repository.rb @@ -28,6 +28,9 @@ class Repository #{REF_PIPELINES} ].freeze + FORMAT_SHA1 = 'sha1' + FORMAT_SHA256 = 'sha256' + include Gitlab::RepositoryCacheAdapter attr_accessor :full_path, :shard, :disk_path, :container, :repo_type diff --git a/app/services/projects/create_service.rb b/app/services/projects/create_service.rb index e0ee3683ac8a06..e4b0fca729af8b 100644 --- a/app/services/projects/create_service.rb +++ b/app/services/projects/create_service.rb @@ -13,6 +13,7 @@ def initialize(user, params) @skip_wiki = @params.delete(:skip_wiki) @initialize_with_sast = Gitlab::Utils.to_boolean(@params.delete(:initialize_with_sast)) @initialize_with_readme = Gitlab::Utils.to_boolean(@params.delete(:initialize_with_readme)) + @use_sha256_repository = Gitlab::Utils.to_boolean(@params.delete(:use_sha256_repository)) && Feature.enabled?(:support_sha256_repositories, user) @import_data = @params.delete(:import_data) @relations_block = @params.delete(:relations_block) @default_branch = @params.delete(:default_branch) @@ -212,6 +213,10 @@ def create_sast_commit ::Security::CiConfiguration::SastCreateService.new(@project, current_user, { initialize_with_sast: true }, commit_on_default: true).execute end + def repository_object_format + @use_sha256_repository ? Repository::FORMAT_SHA256 : Repository::FORMAT_SHA1 + end + def readme_content readme_attrs = { default_branch: default_branch @@ -242,7 +247,7 @@ def save_project_and_import_data next if @project.import? - unless @project.create_repository(default_branch: default_branch) + unless @project.create_repository(default_branch: default_branch, object_format: repository_object_format) raise 'Failed to create repository' end end diff --git a/app/views/projects/_new_project_fields.html.haml b/app/views/projects/_new_project_fields.html.haml index ca1fef6eb32ce2..73c787bd726bc5 100644 --- a/app/views/projects/_new_project_fields.html.haml +++ b/app/views/projects/_new_project_fields.html.haml @@ -3,6 +3,7 @@ - hide_init_with_readme = local_assigns.fetch(:hide_init_with_readme, false) - include_description = local_assigns.fetch(:include_description, true) - track_label = local_assigns.fetch(:track_label, 'blank_project') +- display_sha256_repository = Feature.enabled?(:support_sha256_repositories, current_user) .row{ id: project_name_id } = f.hidden_field :ci_cd_only, value: ci_cd_only @@ -95,6 +96,14 @@ = s_('ProjectsNew|Analyze your source code for known security vulnerabilities.') = link_to _('Learn more.'), help_page_path('user/application_security/sast/index'), target: '_blank', rel: 'noopener noreferrer', data: { track_action: 'followed' } + - if display_sha256_repository + .form-group + = render Pajamas::CheckboxTagComponent.new(name: 'project[use_sha256_repository]') do |c| + - c.with_label do + = s_('ProjectsNew|Use SHA-256 as the repository hashing algorithm') + - c.with_help_text do + = s_('ProjectsNew|Default hashing algorithm is SHA-1.') + -# this partial is from JiHu, see details in https://jihulab.com/gitlab-cn/gitlab/-/merge_requests/675 = render_if_exists 'shared/other_project_options', f: f, visibility_level: visibility_level, track_label: track_label = f.submit _('Create project'), class: "js-create-project-button", data: { testid: 'project-create-button', track_label: "#{track_label}", track_action: "click_button", track_property: "create_project", track_value: "" }, pajamas_button: true diff --git a/config/feature_flags/development/support_sha256_repositories.yml b/config/feature_flags/development/support_sha256_repositories.yml new file mode 100644 index 00000000000000..0482c5dd97c3a7 --- /dev/null +++ b/config/feature_flags/development/support_sha256_repositories.yml @@ -0,0 +1,8 @@ +--- +name: support_sha256_repositories +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/136681 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/431864 +milestone: '16.7' +type: development +group: group::source code +default_enabled: false diff --git a/lib/gitlab/git/repository.rb b/lib/gitlab/git/repository.rb index db6e6b4d00b2b3..087c82ac33956e 100644 --- a/lib/gitlab/git/repository.rb +++ b/lib/gitlab/git/repository.rb @@ -102,9 +102,9 @@ def exists? gitaly_repository_client.exists? end - def create_repository(default_branch = nil) + def create_repository(default_branch = nil, object_format: nil) wrapped_gitaly_errors do - gitaly_repository_client.create_repository(default_branch) + gitaly_repository_client.create_repository(default_branch, object_format: object_format) rescue GRPC::AlreadyExists => e raise RepositoryExists, e.message end diff --git a/lib/gitlab/gitaly_client/repository_service.rb b/lib/gitlab/gitaly_client/repository_service.rb index 457380615f7315..3ba8d7a1afd21a 100644 --- a/lib/gitlab/gitaly_client/repository_service.rb +++ b/lib/gitlab/gitaly_client/repository_service.rb @@ -114,8 +114,8 @@ def fetch_remote(url, refmap:, ssh_auth:, forced:, no_tags:, timeout:, prune: tr end # rubocop: enable Metrics/ParameterLists - def create_repository(default_branch = nil) - request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: encode_binary(default_branch)) + def create_repository(default_branch = nil, object_format: nil) + request = Gitaly::CreateRepositoryRequest.new(repository: @gitaly_repo, default_branch: encode_binary(default_branch), object_format: gitaly_object_format(object_format)) gitaly_client_call(@storage, :repository_service, :create_repository, request, timeout: GitalyClient.fast_timeout) end @@ -449,6 +449,15 @@ def build_set_config_entry(key, value) entry end + + def gitaly_object_format(format) + case format + when Repository::FORMAT_SHA1 + Gitaly::ObjectFormat::OBJECT_FORMAT_SHA1 + when Repository::FORMAT_SHA256 + Gitaly::ObjectFormat::OBJECT_FORMAT_SHA256 + end + end end end end diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 58d123638b163d..00c4e37c00bcbe 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -38103,6 +38103,9 @@ msgstr "" msgid "ProjectsNew|Create new project" msgstr "" +msgid "ProjectsNew|Default hashing algorithm is SHA-1." +msgstr "" + msgid "ProjectsNew|Description format" msgstr "" @@ -38166,6 +38169,9 @@ msgstr "" msgid "ProjectsNew|Unable to suggest a path. Please refresh and try again." msgstr "" +msgid "ProjectsNew|Use SHA-256 as the repository hashing algorithm" +msgstr "" + msgid "ProjectsNew|Visibility Level" msgstr "" diff --git a/spec/factories/projects.rb b/spec/factories/projects.rb index 0962c2513823f8..644a9411ca6d37 100644 --- a/spec/factories/projects.rb +++ b/spec/factories/projects.rb @@ -241,10 +241,11 @@ trait :custom_repo do transient do files { {} } + object_format { Repository::FORMAT_SHA1 } end after :create do |project, evaluator| - raise "Failed to create repository!" unless project.repository.exists? || project.create_repository + raise "Failed to create repository!" unless project.repository.exists? || project.create_repository(object_format: evaluator.object_format) evaluator.files.each do |filename, content| project.repository.create_file( @@ -381,8 +382,12 @@ end trait :empty_repo do - after(:create) do |project| - raise "Failed to create repository!" unless project.create_repository + transient do + object_format { Repository::FORMAT_SHA1 } + end + + after(:create) do |project, evaluator| + raise "Failed to create repository!" unless project.create_repository(object_format: evaluator.object_format) end end diff --git a/spec/features/projects/user_creates_project_spec.rb b/spec/features/projects/user_creates_project_spec.rb index 484808dcfd1ed8..a590d637801e2e 100644 --- a/spec/features/projects/user_creates_project_spec.rb +++ b/spec/features/projects/user_creates_project_spec.rb @@ -55,6 +55,45 @@ expect(page).to have_content('README.md Initial commit') end + context 'when creating a project with SHA256 repository' do + let(:sha256_field) { 'Use SHA-256 as the repository hashing algorithm' } + + it 'creates a new project' do + visit(new_project_path) + + click_link 'Create blank project' + fill_in(:project_name, with: 'With initial commits') + + expect(page).to have_checked_field 'Initialize repository with a README' + expect(page).to have_unchecked_field sha256_field + + check sha256_field + + page.within('#content-body') do + click_button('Create project') + end + + project = Project.last + + expect(page).to have_current_path(project_path(project), ignore_query: true) + expect(page).to have_content('With initial commits') + end + + context 'when "support_sha256_repositories" feature flag is disabled' do + before do + stub_feature_flags(support_sha256_repositories: false) + end + + it 'does not display a SHA256 option' do + visit(new_project_path) + + click_link 'Create blank project' + + expect(page).not_to have_content(sha256_field) + end + end + end + context 'in a subgroup they do not own' do let(:parent) { create(:group) } let!(:subgroup) { create(:group, parent: parent) } diff --git a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb index 727bf494ee6fa9..7677a8287bde2b 100644 --- a/spec/lib/gitlab/gitaly_client/repository_service_spec.rb +++ b/spec/lib/gitlab/gitaly_client/repository_service_spec.rb @@ -355,6 +355,40 @@ client.create_repository('feature/新機能') end + + context 'when object format is provided' do + before do + expect_any_instance_of(Gitaly::RepositoryService::Stub) + .to receive(:create_repository) + .with(gitaly_request_with_path(storage_name, relative_path) + .and(gitaly_request_with_params(default_branch: '', object_format: expected_format)), kind_of(Hash)) + .and_return(double) + end + + context 'with SHA1 format' do + let(:expected_format) { :OBJECT_FORMAT_SHA1 } + + it 'sends a create_repository message with object format' do + client.create_repository(object_format: Repository::FORMAT_SHA1) + end + end + + context 'with SHA256 format' do + let(:expected_format) { :OBJECT_FORMAT_SHA256 } + + it 'sends a create_repository message with object format' do + client.create_repository(object_format: Repository::FORMAT_SHA256) + end + end + + context 'with unknown format' do + let(:expected_format) { :OBJECT_FORMAT_UNSPECIFIED } + + it 'sends a create_repository message with object format' do + client.create_repository(object_format: 'unknown') + end + end + end end describe '#raw_changes_between' do diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb index 5f1054a35bbae8..e6905dbc188f3e 100644 --- a/spec/models/project_spec.rb +++ b/spec/models/project_spec.rb @@ -3150,7 +3150,7 @@ def has_external_wiki end it 'passes through default branch' do - expect(project.repository).to receive(:create_repository).with('pineapple') + expect(project.repository).to receive(:create_repository).with('pineapple', object_format: nil) expect(project.create_repository(default_branch: 'pineapple')).to eq(true) end @@ -3164,6 +3164,13 @@ def has_external_wiki project.create_repository end end + + context 'using a SHA256 repository' do + it 'creates the repository' do + expect(project.repository).to receive(:create_repository).with(nil, object_format: Repository::FORMAT_SHA256) + expect(project.create_repository(object_format: Repository::FORMAT_SHA256)).to eq(true) + end + end end describe '#ensure_repository' do diff --git a/spec/services/projects/create_service_spec.rb b/spec/services/projects/create_service_spec.rb index ce7e5188c7bde8..2013e0d2690990 100644 --- a/spec/services/projects/create_service_spec.rb +++ b/spec/services/projects/create_service_spec.rb @@ -812,6 +812,31 @@ def wiki_repo(project) end end + context 'when SHA256 format is requested' do + let(:project) { create_project(user, opts) } + let(:opts) { super().merge(initialize_with_readme: true, use_sha256_repository: true) } + + before do + allow(Gitlab::CurrentSettings).to receive(:default_branch_name).and_return('main') + end + + it 'creates a repository with SHA256 commit hashes', :aggregate_failures do + expect(project.repository.commit_count).to be(1) + expect(project.commit.id.size).to eq 64 + end + + context 'when "support_sha256_repositories" feature flag is disabled' do + before do + stub_feature_flags(support_sha256_repositories: false) + end + + it 'creates a repository with default SHA1 commit hash' do + expect(project.repository.commit_count).to be(1) + expect(project.commit.id.size).to eq 40 + end + end + end + describe 'create integration for the project' do subject(:project) { create_project(user, opts) } -- GitLab