From c5d9f004dae9d294746d4dcb7a86605ff8d570b4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= Date: Fri, 20 Mar 2020 14:41:25 +0100 Subject: [PATCH 1/2] Introduced Ndjson writer - Fix ndjson and legacy writer specs - Fix root key to be projects instead of project - Fix EE specs - Move common methods in common_util - Support for new ndjson structure - Fix ndjson writer - Fixed context for JSON and Ndjson - Refactor tree_saver_spec - Remove unused attr_readers - Fix specs for ndjson writer - Added shared example for tree saver - Rename shared example for tree restorer - Moved common methods to common utils - Added missing specs for ndjson_writer - Fix specs with feature flag - Add frozen string literal - Fix write file in initializer- Add spec for tree_saver spec - Fix Tree Saver to use full_path --- .../import_export/project/tree_saver_spec.rb | 92 ++---- .../project/tree_saver_shared_example.rb | 65 ++++ .../import_export/json/ndjson_writer.rb | 59 ++++ .../import_export/project/tree_saver.rb | 18 +- .../import_export/export_file_spec.rb | 73 ++-- spec/lib/gitlab/import_export/fork_spec.rb | 2 + .../import_export_equivalence_spec.rb | 4 + .../import_export/json/ndjson_writer_spec.rb | 63 ++++ .../import_export/project/tree_saver_spec.rb | 311 ++---------------- spec/support/import_export/common_util.rb | 42 +++ .../project/export_file_shared_examples.rb | 50 +++ .../tree_restorer_shared_examples.rb} | 0 .../project/tree_saver_shared_examples.rb | 272 +++++++++++++++ 13 files changed, 664 insertions(+), 387 deletions(-) create mode 100644 ee/spec/support/shared_examples/lib/ee/gitlab/import_export/project/tree_saver_shared_example.rb create mode 100644 lib/gitlab/import_export/json/ndjson_writer.rb create mode 100644 spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb create mode 100644 spec/support/shared_examples/lib/gitlab/import_export/project/export_file_shared_examples.rb rename spec/support/shared_examples/lib/gitlab/import_export/{project_tree_restorer_shared_examples.rb => project/tree_restorer_shared_examples.rb} (100%) create mode 100644 spec/support/shared_examples/lib/gitlab/import_export/project/tree_saver_shared_examples.rb diff --git a/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb b/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb index dc21a2f92ecc2f..74195916c4f673 100644 --- a/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb +++ b/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb @@ -3,83 +3,35 @@ require 'spec_helper' describe Gitlab::ImportExport::Project::TreeSaver do - describe 'saves the project tree into a json object' do - let_it_be(:user) { create(:user) } - let_it_be(:group) { create(:group) } - let_it_be(:project) { create(:project, group: group) } - let_it_be(:issue) { create(:issue, project: project) } + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:project) { create(:project, group: group) } + let_it_be(:issue) { create(:issue, project: project) } - let_it_be(:design) { create(:design, :with_file, versions_count: 2, issue: issue) } - let_it_be(:note) { create(:diff_note_on_design, noteable: design, project: project, author: user) } - let_it_be(:note2) { create(:note, noteable: issue, project: project, author: user) } + let_it_be(:design) { create(:design, :with_file, versions_count: 2, issue: issue) } + let_it_be(:note) { create(:diff_note_on_design, noteable: design, project: project, author: user) } + let_it_be(:note2) { create(:note, noteable: issue, project: project, author: user) } - let_it_be(:epic) { create(:epic, group: group) } - let_it_be(:epic_issue) { create(:epic_issue, issue: issue, epic: epic) } + let_it_be(:epic) { create(:epic, group: group) } + let_it_be(:epic_issue) { create(:epic_issue, issue: issue, epic: epic) } - let(:shared) { project.import_export_shared } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec_ee" } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } - let(:saved_project_json) do - project_tree_saver.save - project_json(project_tree_saver.full_path) - end + let(:shared) { project.import_export_shared } + let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec_ee" } + let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } - before do - project.add_maintainer(user) - end - - after do - FileUtils.rm_rf(export_path) - end - - it 'saves successfully' do - expect(project_tree_saver.save).to be true - end - - describe 'the designs json' do - let(:issue_json) { saved_project_json['issues'].first } - - it 'saves issue.designs correctly' do - expect(issue_json['designs'].size).to eq(1) - end - - it 'saves issue.design_versions correctly' do - actions = issue_json['design_versions'].map do |v| - v['actions'] - end.flatten - - expect(issue_json['design_versions'].size).to eq(2) - issue_json['design_versions'].each do |version| - expect(version['author_id']).to eq(issue.author_id) - end - expect(actions.size).to eq(2) - actions.each do |action| - expect(action['design']).to be_present - end - end - end - - context 'epics' do - it 'has epic_issue' do - expect(saved_project_json['issues'].first['epic_issue']).not_to be_empty - expect(saved_project_json['issues'].first['epic_issue']['id']).to eql(epic_issue.id) - end - - it 'has epic' do - expect(saved_project_json['issues'].first['epic_issue']['epic']['title']).to eql(epic.title) - end + before do + project.add_maintainer(user) + end - it 'does not have epic_id' do - expect(saved_project_json['issues'].first['epic_issue']['epic_id']).to be_nil - end + after do + FileUtils.rm_rf(export_path) + end - it 'does not have issue_id' do - expect(saved_project_json['issues'].first['epic_issue']['issue_id']).to be_nil - end - end + context 'with JSON' do + it_behaves_like "EE saves project tree successfully", false end - def project_json(filename) - ::JSON.parse(IO.read(filename)) + context 'with NDJSON' do + it_behaves_like "EE saves project tree successfully", true end end diff --git a/ee/spec/support/shared_examples/lib/ee/gitlab/import_export/project/tree_saver_shared_example.rb b/ee/spec/support/shared_examples/lib/ee/gitlab/import_export/project/tree_saver_shared_example.rb new file mode 100644 index 00000000000000..ac55bc53fc0c0e --- /dev/null +++ b/ee/spec/support/shared_examples/lib/ee/gitlab/import_export/project/tree_saver_shared_example.rb @@ -0,0 +1,65 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'EE saves project tree successfully' do |ndjson_enabled| + include ::ImportExport::CommonUtil + + let(:full_path) do + project_tree_saver.save + + if ndjson_enabled == true + File.join(shared.export_path, 'tree') + else + File.join(shared.export_path, Gitlab::ImportExport.project_filename) + end + end + + let(:exportable_path) { 'project' } + + before do + stub_feature_flags(project_export_as_ndjson: ndjson_enabled) + end + + it 'saves successfully' do + expect(project_tree_saver.save).to be true + end + + describe 'the designs json' do + let(:issue_json) { saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first } + + it 'saves issue.designs correctly' do + expect(issue_json['designs'].size).to eq(1) + end + + it 'saves issue.design_versions correctly' do + actions = issue_json['design_versions'].flat_map { |v| v['actions'] } + + expect(issue_json['design_versions'].size).to eq(2) + issue_json['design_versions'].each do |version| + expect(version['author_id']).to eq(issue.author_id) + end + expect(actions.size).to eq(2) + actions.each do |action| + expect(action['design']).to be_present + end + end + end + + context 'epics' do + it 'has epic_issue' do + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']).not_to be_empty + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']['id']).to eql(epic_issue.id) + end + + it 'has epic' do + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']['epic']['title']).to eql(epic.title) + end + + it 'does not have epic_id' do + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']['epic_id']).to be_nil + end + + it 'does not have issue_id' do + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']['issue_id']).to be_nil + end + end +end diff --git a/lib/gitlab/import_export/json/ndjson_writer.rb b/lib/gitlab/import_export/json/ndjson_writer.rb new file mode 100644 index 00000000000000..e74fdd74049f9d --- /dev/null +++ b/lib/gitlab/import_export/json/ndjson_writer.rb @@ -0,0 +1,59 @@ +# frozen_string_literal: true + +module Gitlab + module ImportExport + module JSON + class NdjsonWriter + include Gitlab::ImportExport::CommandLineUtil + + def initialize(dir_path) + @dir_path = dir_path + end + + def close + end + + def write_attributes(exportable_path, hash) + # It will create: + # tree/project.json + with_file("#{exportable_path}.json") do |file| + file.write(hash.to_json) + end + end + + def write_relation(exportable_path, relation, value) + # It will create: + # tree/project/ci_cd_setting.ndjson + with_file(exportable_path, "#{relation}.ndjson") do |file| + file.write(value.to_json) + end + end + + def write_relation_array(exportable_path, relation, items) + # It will create: + # tree/project/merge_requests.ndjson + with_file(exportable_path, "#{relation}.ndjson") do |file| + items.each do |item| + file.write(item.to_json) + file.write("\n") + end + end + end + + private + + def with_file(*path) + file_path = File.join(@dir_path, *path) + raise ArgumentError, "The #{file_path} already exist" if File.exist?(file_path) + + # ensure that path is created + mkdir_p(File.dirname(file_path)) + + File.open(file_path, "wb") do |file| + yield(file) + end + end + end + end + end +end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 988776fe600b32..0017aa523c1b98 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -11,15 +11,9 @@ def initialize(project:, current_user:, shared:, params: {}) @project = project @current_user = current_user @shared = shared - @full_path = File.join(@shared.export_path, ImportExport.project_filename) end def save - json_writer = ImportExport::JSON::LegacyWriter.new( - @full_path, - allowed_path: "project" - ) - ImportExport::JSON::StreamingSerializer.new( exportable, reader.project_tree, @@ -57,6 +51,18 @@ def exportable_params def presenter_class Projects::ImportExport::ProjectExportPresenter end + + def json_writer + @json_writer ||= begin + if ::Feature.enabled?(:project_export_as_ndjson, @project.namespace) + full_path = File.join(@shared.export_path, 'tree') + Gitlab::ImportExport::JSON::NdjsonWriter.new(full_path) + else + full_path = File.join(@shared.export_path, ImportExport.project_filename) + Gitlab::ImportExport::JSON::LegacyWriter.new(full_path, allowed_path: 'project') + end + end + end end end end diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb index 54a6ac1551b0f9..6c560a341cee0d 100644 --- a/spec/features/projects/import_export/export_file_spec.rb +++ b/spec/features/projects/import_export/export_file_spec.rb @@ -38,38 +38,70 @@ sign_in(user) end - it 'exports a project successfully', :sidekiq_might_not_need_inline do - visit edit_project_path(project) + context "with legacy export" do + before do + stub_feature_flags(streaming_serializer: false) + stub_feature_flags(project_export_as_ndjson: false) + end + + it_behaves_like "export file without sensitive words" + end - expect(page).to have_content('Export project') + context "with streaming serializer" do + before do + stub_feature_flags(streaming_serializer: true) + stub_feature_flags(project_export_as_ndjson: false) + end + it_behaves_like "export file without sensitive words" + end - find(:link, 'Export project').send_keys(:return) + context "with ndjson" do + before do + stub_feature_flags(streaming_serializer: true) + stub_feature_flags(project_export_as_ndjson: true) + end - visit edit_project_path(project) + it 'exports a project successfully', :sidekiq_might_not_need_inline do + visit edit_project_path(project) - expect(page).to have_content('Download export') + expect(page).to have_content('Export project') - expect(project.export_status).to eq(:finished) - expect(project.export_file.path).to include('tar.gz') + find(:link, 'Export project').send_keys(:return) - in_directory_with_expanded_export(project) do |exit_status, tmpdir| - expect(exit_status).to eq(0) + visit edit_project_path(project) - project_json_path = File.join(tmpdir, 'project.json') - expect(File).to exist(project_json_path) + expect(page).to have_content('Download export') - project_hash = JSON.parse(IO.read(project_json_path)) + expect(project.export_status).to eq(:finished) + expect(project.export_file.path).to include('tar.gz') - sensitive_words.each do |sensitive_word| - found = find_sensitive_attributes(sensitive_word, project_hash) + in_directory_with_expanded_export(project) do |exit_status, tmpdir| + expect(exit_status).to eq(0) - expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + project_json_path = File.join(tmpdir, 'tree', 'project.json') + expect(File).to exist(project_json_path) + + relations = [] + relations << JSON.parse(IO.read(project_json_path)) + Dir.glob(File.join(tmpdir, 'tree/project', '*.ndjson')) do |rb_filename| + File.foreach(rb_filename) do |line| + json = ActiveSupport::JSON.decode(line) + relations << json + end + end + + relations.each do |relation_hash| + sensitive_words.each do |sensitive_word| + found = find_sensitive_attributes(sensitive_word, relation_hash) + + expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + end + end end end - end - def failure_message(key_found, parent, sensitive_word) - <<-MSG + def failure_message(key_found, parent, sensitive_word) + <<-MSG Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. @@ -82,7 +114,8 @@ def failure_message(key_found, parent, sensitive_word) IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} CURRENT_SPEC: #{__FILE__} - MSG + MSG + end end end end diff --git a/spec/lib/gitlab/import_export/fork_spec.rb b/spec/lib/gitlab/import_export/fork_spec.rb index 8aa28353c044d9..bb79331efac2c7 100644 --- a/spec/lib/gitlab/import_export/fork_spec.rb +++ b/spec/lib/gitlab/import_export/fork_spec.rb @@ -32,6 +32,8 @@ end before do + stub_feature_flags(project_export_as_ndjson: false) + allow_next_instance_of(Gitlab::ImportExport) do |instance| allow(instance).to receive(:storage_path).and_return(export_path) end diff --git a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb index 50b26637cb1463..707975f20b649c 100644 --- a/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb +++ b/spec/lib/gitlab/import_export/import_export_equivalence_spec.rb @@ -20,6 +20,10 @@ let(:json_fixture) { 'complex' } + before do + stub_feature_flags(project_export_as_ndjson: false) + end + it 'yields the initial tree when importing and exporting it again' do project = create(:project, creator: create(:user, :admin)) diff --git a/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb new file mode 100644 index 00000000000000..1b14141ddbae0b --- /dev/null +++ b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb @@ -0,0 +1,63 @@ +# frozen_string_literal: true + +require "spec_helper" + +describe Gitlab::ImportExport::JSON::NdjsonWriter do + include ImportExport::CommonUtil + + let(:path) { "#{Dir.tmpdir}/legacy_writer_spec/tree" } + let(:exportable_path) { 'projects' } + + subject { described_class.new(path) } + + after do + FileUtils.rm_rf(path) + end + + describe "#write_attributes" do + it "writes correct json to root" do + expected_hash = { "key" => "value_1", "key_1" => "value_2" } + subject.write_attributes(exportable_path, expected_hash) + + expect(consume_attributes(path, exportable_path)).to eq(expected_hash) + end + end + + describe "#write_relation" do + context "when single relation is serialized" do + it "appends json in correct file " do + relation = "relation" + value = { "key" => "value_1", "key_1" => "value_1" } + subject.write_relation(exportable_path, relation, value) + + expect(consume_relations(path, exportable_path, relation)).to eq([value]) + end + end + + context "when single relation is already serialized" do + it "raise exception" do + values = [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }] + relation = "relation" + file_path = File.join(path, exportable_path, "#{relation}.ndjson") + subject.write_relation(exportable_path, relation, values[0]) + + expect {subject.write_relation(exportable_path, relation, values[1])}.to raise_exception("The #{file_path} already exist") + end + end + end + + describe "#write_relation_array" do + it "writes json in correct files" do + values = [{ "key" => "value_1", "key_1" => "value_1" }, { "key" => "value_2", "key_1" => "value_2" }] + relations = %w(relation1 relation2) + relations.each do |relation| + subject.write_relation_array(exportable_path, relation, values.to_enum) + end + subject.close + + relations.each do |relation| + expect(consume_relations(path, exportable_path, relation)).to eq(values) + end + end + end +end diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index c910ee5430b45d..d1da7236824ef7 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -3,296 +3,29 @@ require 'spec_helper' describe Gitlab::ImportExport::Project::TreeSaver do - describe 'saves the project tree into a json object' do - let(:shared) { project.import_export_shared } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } - let(:user) { create(:user) } - let!(:project) { setup_project } - - before do - project.add_maintainer(user) - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - allow_any_instance_of(MergeRequest).to receive(:source_branch_sha).and_return('ABCD') - allow_any_instance_of(MergeRequest).to receive(:target_branch_sha).and_return('DCBA') - end - - after do - FileUtils.rm_rf(export_path) - end - - it 'saves project successfully' do - expect(project_tree_saver.save).to be true - end - - context 'JSON' do - let(:saved_project_json) do - project_tree_saver.save - project_json(project_tree_saver.full_path) - end - - # It is not duplicated in - # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` - context 'with description override' do - let(:params) { { description: 'Foo Bar' } } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared, params: params) } - - it 'overrides the project description' do - expect(saved_project_json).to include({ 'description' => params[:description] }) - end - end - - it 'saves the correct json' do - expect(saved_project_json).to include({ 'description' => 'description', 'visibility_level' => 20 }) - end - - it 'has approvals_before_merge set' do - expect(saved_project_json['approvals_before_merge']).to eq(1) - end - - it 'has milestones' do - expect(saved_project_json['milestones']).not_to be_empty - end - - it 'has merge requests' do - expect(saved_project_json['merge_requests']).not_to be_empty - end - - it 'has merge request\'s milestones' do - expect(saved_project_json['merge_requests'].first['milestone']).not_to be_empty - end - - it 'has merge request\'s source branch SHA' do - expect(saved_project_json['merge_requests'].first['source_branch_sha']).to eq('ABCD') - end - - it 'has merge request\'s target branch SHA' do - expect(saved_project_json['merge_requests'].first['target_branch_sha']).to eq('DCBA') - end - - it 'has events' do - expect(saved_project_json['merge_requests'].first['milestone']['events']).not_to be_empty - end - - it 'has snippets' do - expect(saved_project_json['snippets']).not_to be_empty - end - - it 'has snippet notes' do - expect(saved_project_json['snippets'].first['notes']).not_to be_empty - end - - it 'has releases' do - expect(saved_project_json['releases']).not_to be_empty - end - - it 'has no author on releases' do - expect(saved_project_json['releases'].first['author']).to be_nil - end - - it 'has the author ID on releases' do - expect(saved_project_json['releases'].first['author_id']).not_to be_nil - end - - it 'has issues' do - expect(saved_project_json['issues']).not_to be_empty - end - - it 'has issue comments' do - notes = saved_project_json['issues'].first['notes'] - - expect(notes).not_to be_empty - expect(notes.first['type']).to eq('DiscussionNote') - end - - it 'has issue assignees' do - expect(saved_project_json['issues'].first['issue_assignees']).not_to be_empty - end - - it 'has author on issue comments' do - expect(saved_project_json['issues'].first['notes'].first['author']).not_to be_empty - end - - it 'has system note metadata on issue comments' do - metadata = saved_project_json['issues'].first['notes'].first['system_note_metadata'] - - expect(metadata['action']).to eq('description') - end - - it 'has project members' do - expect(saved_project_json['project_members']).not_to be_empty - end - - it 'has merge requests diffs' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']).not_to be_empty - end - - it 'has merge request diff files' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_files']).not_to be_empty - end - - it 'has merge request diff commits' do - expect(saved_project_json['merge_requests'].first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty - end - - it 'has merge requests comments' do - expect(saved_project_json['merge_requests'].first['notes']).not_to be_empty - end - - it 'has author on merge requests comments' do - expect(saved_project_json['merge_requests'].first['notes'].first['author']).not_to be_empty - end - - it 'has system note metadata on merge requests comments' do - metadata = saved_project_json['merge_requests'].first['notes'].first['system_note_metadata'] - - expect(metadata['commit_count']).to eq(1) - expect(metadata['action']).to eq('commit') - end - - it 'has pipeline stages' do - expect(saved_project_json.dig('ci_pipelines', 0, 'stages')).not_to be_empty - end - - it 'has pipeline statuses' do - expect(saved_project_json.dig('ci_pipelines', 0, 'stages', 0, 'statuses')).not_to be_empty - end - - it 'has pipeline builds' do - builds_count = saved_project_json - .dig('ci_pipelines', 0, 'stages', 0, 'statuses') - .count { |hash| hash['type'] == 'Ci::Build' } - - expect(builds_count).to eq(1) - end - - it 'has no when YML attributes but only the DB column' do - expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) - - saved_project_json - end - - it 'has pipeline commits' do - expect(saved_project_json['ci_pipelines']).not_to be_empty - end - - it 'has ci pipeline notes' do - expect(saved_project_json['ci_pipelines'].first['notes']).not_to be_empty - end - - it 'has labels with no associations' do - expect(saved_project_json['labels']).not_to be_empty - end - - it 'has labels associated to records' do - expect(saved_project_json['issues'].first['label_links'].first['label']).not_to be_empty - end - - it 'has project and group labels' do - label_types = saved_project_json['issues'].first['label_links'].map { |link| link['label']['type'] } - - expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) - end - - it 'has priorities associated to labels' do - priorities = saved_project_json['issues'].first['label_links'].flat_map { |link| link['label']['priorities'] } - - expect(priorities).not_to be_empty - end - - it 'has issue resource label events' do - expect(saved_project_json['issues'].first['resource_label_events']).not_to be_empty - end - - it 'has merge request resource label events' do - expect(saved_project_json['merge_requests'].first['resource_label_events']).not_to be_empty - end - - it 'saves the correct service type' do - expect(saved_project_json['services'].first['type']).to eq('CustomIssueTrackerService') - end - - it 'saves the properties for a service' do - expect(saved_project_json['services'].first['properties']).to eq('one' => 'value') - end - - it 'has project feature' do - project_feature = saved_project_json['project_feature'] - expect(project_feature).not_to be_empty - expect(project_feature["issues_access_level"]).to eq(ProjectFeature::DISABLED) - expect(project_feature["wiki_access_level"]).to eq(ProjectFeature::ENABLED) - expect(project_feature["builds_access_level"]).to eq(ProjectFeature::PRIVATE) - end - - it 'has custom attributes' do - expect(saved_project_json['custom_attributes'].count).to eq(2) - end - - it 'has badges' do - expect(saved_project_json['project_badges'].count).to eq(2) - end - - it 'does not complain about non UTF-8 characters in MR diff files' do - ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") - - expect(project_tree_saver.save).to be true - end - - context 'group members' do - let(:user2) { create(:user, email: 'group@member.com') } - let(:member_emails) do - saved_project_json['project_members'].map do |pm| - pm['user']['email'] - end - end - - before do - Group.first.add_developer(user2) - end - - it 'does not export group members if it has no permission' do - Group.first.add_developer(user) - - expect(member_emails).not_to include('group@member.com') - end - - it 'does not export group members as maintainer' do - Group.first.add_maintainer(user) - - expect(member_emails).not_to include('group@member.com') - end - - it 'exports group members as group owner' do - Group.first.add_owner(user) - - expect(member_emails).to include('group@member.com') - end - - context 'as admin' do - let(:user) { create(:admin) } - - it 'exports group members as admin' do - expect(member_emails).to include('group@member.com') - end - - it 'exports group members as project members' do - member_types = saved_project_json['project_members'].map { |pm| pm['source_type'] } + let(:shared) { project.import_export_shared } + let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } + let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } + let(:user) { create(:user) } + let!(:project) { setup_project } + + before do + project.add_maintainer(user) + allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) + allow_any_instance_of(MergeRequest).to receive(:source_branch_sha).and_return('ABCD') + allow_any_instance_of(MergeRequest).to receive(:target_branch_sha).and_return('DCBA') + end - expect(member_types).to all(eq('Project')) - end - end - end + after do + FileUtils.rm_rf(export_path) + end - context 'project attributes' do - it 'does not contain the runners token' do - expect(saved_project_json).not_to include("runners_token" => 'token') - end - end + context 'with JSON' do + it_behaves_like "saves project tree successfully", false + end - it 'has a board and a list' do - expect(saved_project_json['boards'].first['lists']).not_to be_empty - end - end + context 'with NDJSON' do + it_behaves_like "saves project tree successfully", true end def setup_project @@ -355,8 +88,4 @@ def setup_project project end - - def project_json(filename) - ::JSON.parse(IO.read(filename)) - end end diff --git a/spec/support/import_export/common_util.rb b/spec/support/import_export/common_util.rb index 9281937e4bac4a..da5a537911263d 100644 --- a/spec/support/import_export/common_util.rb +++ b/spec/support/import_export/common_util.rb @@ -26,6 +26,23 @@ def test_tmp_path "tmp/tests/gitlab-test/import_export" end + def saved_relations(path, exportable_path, key, ndjson_enabled) + if ndjson_enabled == true + json = if key == :projects + consume_attributes(path, exportable_path) + else + consume_relations(path, exportable_path, key) + end + + json = json.first if key == :project_feature + else + json = project_json(path) + json = json[key.to_s] unless key == :projects + end + + json + end + def restore_then_save_project(project, import_path:, export_path:) project_restorer = get_project_restorer(project, import_path) project_saver = get_project_saver(project, export_path) @@ -50,5 +67,30 @@ def get_shared_env(path:) allow(shared).to receive(:export_path).and_return(path) end end + + def consume_attributes(dir_path, exportable_path) + path = File.join(dir_path, "#{exportable_path}.json") + return unless File.exist?(path) + + ActiveSupport::JSON.decode(IO.read(path)) + end + + def consume_relations(dir_path, exportable_path, key) + path = File.join(dir_path, exportable_path, "#{key}.ndjson") + return unless File.exist?(path) + + relations = [] + + File.foreach(path) do |line| + json = ActiveSupport::JSON.decode(line) + relations << json + end + + relations.flatten + end + + def project_json(filename) + ::JSON.parse(IO.read(filename)) + end end end diff --git a/spec/support/shared_examples/lib/gitlab/import_export/project/export_file_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/project/export_file_shared_examples.rb new file mode 100644 index 00000000000000..649967ed56a51e --- /dev/null +++ b/spec/support/shared_examples/lib/gitlab/import_export/project/export_file_shared_examples.rb @@ -0,0 +1,50 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'export file without sensitive words' do + it 'exports a project successfully', :sidekiq_might_not_need_inline do + visit edit_project_path(project) + + expect(page).to have_content('Export project') + + find(:link, 'Export project').send_keys(:return) + + visit edit_project_path(project) + + expect(page).to have_content('Download export') + + expect(project.export_status).to eq(:finished) + expect(project.export_file.path).to include('tar.gz') + + in_directory_with_expanded_export(project) do |exit_status, tmpdir| + expect(exit_status).to eq(0) + + project_json_path = File.join(tmpdir, 'project.json') + expect(File).to exist(project_json_path) + + project_hash = JSON.parse(IO.read(project_json_path)) + + sensitive_words.each do |sensitive_word| + found = find_sensitive_attributes(sensitive_word, project_hash) + + expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + end + end + end + + def failure_message(key_found, parent, sensitive_word) + <<-MSG + Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} + + If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. + + Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the + correspondent hash or model as the value. + + Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be + reset (to prevent duplicate column problems while importing to the same instance). + + IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} + CURRENT_SPEC: #{__FILE__} + MSG + end +end diff --git a/spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_restorer_shared_examples.rb similarity index 100% rename from spec/support/shared_examples/lib/gitlab/import_export/project_tree_restorer_shared_examples.rb rename to spec/support/shared_examples/lib/gitlab/import_export/project/tree_restorer_shared_examples.rb diff --git a/spec/support/shared_examples/lib/gitlab/import_export/project/tree_saver_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_saver_shared_examples.rb new file mode 100644 index 00000000000000..5fd1bd9baf9ce7 --- /dev/null +++ b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_saver_shared_examples.rb @@ -0,0 +1,272 @@ +# frozen_string_literal: true + +RSpec.shared_examples 'saves project tree successfully' do |ndjson_enabled| + include ImportExport::CommonUtil + + let(:full_path) do + project_tree_saver.save + + if ndjson_enabled == true + File.join(shared.export_path, 'tree') + else + File.join(shared.export_path, Gitlab::ImportExport.project_filename) + end + end + + let(:exportable_path) { 'project' } + + before do + stub_feature_flags(project_export_as_ndjson: ndjson_enabled) + end + + it 'saves project successfully' do + expect(project_tree_saver.save).to be true + end + # It is not duplicated indoes not contain the runners token + # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` + context 'with description override' do + let(:params) { { description: 'Foo Bar' } } + let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared, params: params) } + + it 'overrides the project description' do + expect(saved_relations(full_path, exportable_path, :projects, ndjson_enabled)).to include({ 'description' => params[:description] }) + end + end + + it 'saves the correct json' do + expect(saved_relations(full_path, exportable_path, :projects, ndjson_enabled)).to include({ 'description' => 'description', 'visibility_level' => 20 }) + end + + it 'has approvals_before_merge set' do + expect(saved_relations(full_path, exportable_path, :projects, ndjson_enabled)['approvals_before_merge']).to eq(1) + end + + it 'has milestones' do + expect(saved_relations(full_path, exportable_path, :milestones, ndjson_enabled)).not_to be_empty + end + + it 'has merge requests' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled)).not_to be_empty + end + + it 'has merge request\'s milestones' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['milestone']).not_to be_empty + end + it 'has merge request\'s source branch SHA' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['source_branch_sha']).to eq('ABCD') + end + + it 'has merge request\'s target branch SHA' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['target_branch_sha']).to eq('DCBA') + end + + it 'has events' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['milestone']['events']).not_to be_empty + end + + it 'has snippets' do + expect(saved_relations(full_path, exportable_path, :snippets, ndjson_enabled)).not_to be_empty + end + + it 'has snippet notes' do + expect(saved_relations(full_path, exportable_path, :snippets, ndjson_enabled).first['notes']).not_to be_empty + end + + it 'has releases' do + expect(saved_relations(full_path, exportable_path, :releases, ndjson_enabled)).not_to be_empty + end + + it 'has no author on releases' do + expect(saved_relations(full_path, exportable_path, :releases, ndjson_enabled).first['author']).to be_nil + end + + it 'has the author ID on releases' do + expect(saved_relations(full_path, exportable_path, :releases, ndjson_enabled).first['author_id']).not_to be_nil + end + + it 'has issues' do + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled)).not_to be_empty + end + + it 'has issue comments' do + notes = saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['notes'] + + expect(notes).not_to be_empty + expect(notes.first['type']).to eq('DiscussionNote') + end + + it 'has issue assignees' do + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['issue_assignees']).not_to be_empty + end + + it 'has author on issue comments' do + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['notes'].first['author']).not_to be_empty + end + + it 'has project members' do + expect(saved_relations(full_path, exportable_path, :project_members, ndjson_enabled)).not_to be_empty + end + + it 'has merge requests diffs' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['merge_request_diff']).not_to be_empty + end + + it 'has merge request diff files' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['merge_request_diff']['merge_request_diff_files']).not_to be_empty + end + + it 'has merge request diff commits' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty + end + + it 'has merge requests comments' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['notes']).not_to be_empty + end + + it 'has author on merge requests comments' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['notes'].first['author']).not_to be_empty + end + + it 'has pipeline stages' do + expect(saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled).dig(0, 'stages')).not_to be_empty + end + + it 'has pipeline statuses' do + expect(saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled).dig(0, 'stages', 0, 'statuses')).not_to be_empty + end + + it 'has pipeline builds' do + builds_count = saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled).dig(0, 'stages', 0, 'statuses') + .count { |hash| hash['type'] == 'Ci::Build' } + + expect(builds_count).to eq(1) + end + + it 'has no when YML attributes but only the DB column' do + expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) + + project_tree_saver.save + end + + it 'has pipeline commits' do + expect(saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled)).not_to be_empty + end + + it 'has ci pipeline notes' do + expect(saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled).first['notes']).not_to be_empty + end + + it 'has labels with no associations' do + expect(saved_relations(full_path, exportable_path, :labels, ndjson_enabled)).not_to be_empty + end + + it 'has labels associated to records' do + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['label_links'].first['label']).not_to be_empty + end + + it 'has project and group labels' do + label_types = saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['label_links'].map { |link| link['label']['type'] } + + expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) + end + + it 'has priorities associated to labels' do + priorities = saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['label_links'].flat_map { |link| link['label']['priorities'] } + + expect(priorities).not_to be_empty + end + + it 'has issue resource label events' do + expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['resource_label_events']).not_to be_empty + end + + it 'has merge request resource label events' do + expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['resource_label_events']).not_to be_empty + end + + it 'saves the correct service type' do + expect(saved_relations(full_path, exportable_path, :services, ndjson_enabled).first['type']).to eq('CustomIssueTrackerService') + end + + it 'saves the properties for a service' do + expect(saved_relations(full_path, exportable_path, :services, ndjson_enabled).first['properties']).to eq('one' => 'value') + end + + it 'has project feature' do + project_feature = saved_relations(full_path, exportable_path, :project_feature, ndjson_enabled) + expect(project_feature).not_to be_empty + expect(project_feature["issues_access_level"]).to eq(ProjectFeature::DISABLED) + expect(project_feature["wiki_access_level"]).to eq(ProjectFeature::ENABLED) + expect(project_feature["builds_access_level"]).to eq(ProjectFeature::PRIVATE) + end + + it 'has custom attributes' do + expect(saved_relations(full_path, exportable_path, :custom_attributes, ndjson_enabled).count).to eq(2) + end + + it 'has badges' do + expect(saved_relations(full_path, exportable_path, :project_badges, ndjson_enabled).count).to eq(2) + end + + it 'does not complain about non UTF-8 characters in MR diff files' do + ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") + + expect(project_tree_saver.save).to be true + end + + context 'group members' do + let(:user2) { create(:user, email: 'group@member.com') } + let(:member_emails) do + emails = saved_relations(full_path, exportable_path, :project_members, ndjson_enabled).map do |pm| + pm['user']['email'] + end + emails + end + + before do + Group.first.add_developer(user2) + end + + it 'does not export group members if it has no permission' do + Group.first.add_developer(user) + + expect(member_emails).not_to include('group@member.com') + end + + it 'does not export group members as maintainer' do + Group.first.add_maintainer(user) + + expect(member_emails).not_to include('group@member.com') + end + + it 'exports group members as group owner' do + Group.first.add_owner(user) + + expect(member_emails).to include('group@member.com') + end + + context 'as admin' do + let(:user) { create(:admin) } + + it 'exports group members as admin' do + expect(member_emails).to include('group@member.com') + end + + it 'exports group members as project members' do + member_types = saved_relations(full_path, exportable_path, :project_members, ndjson_enabled).map { |pm| pm['source_type'] } + + expect(member_types).to all(eq('Project')) + end + end + end + + context 'project attributes' do + it 'does not contain the runners token' do + expect(saved_relations(full_path, exportable_path, :projects, ndjson_enabled)).not_to include("runners_token" => 'token') + end + end + + it 'has a board and a list' do + expect(saved_relations(full_path, exportable_path, :boards, ndjson_enabled).first['lists']).not_to be_empty + end +end -- GitLab From 02b1b8b8ca10bfc3b0ac07397ca700e9d8de007a Mon Sep 17 00:00:00 2001 From: nmilojevic1 Date: Fri, 3 Apr 2020 12:49:26 +0200 Subject: [PATCH 2/2] Refactor specs for ndjson - Make specs faster using let_it_be - Move shared_examples to spec file - Apply MR suggestions --- .../import_export/project/tree_saver_spec.rb | 71 ++- .../project/tree_saver_shared_example.rb | 65 --- .../import_export/export_file_spec.rb | 77 ++-- .../import_export/json/ndjson_writer_spec.rb | 2 +- .../import_export/project/tree_saver_spec.rb | 408 ++++++++++++++++-- spec/support/import_export/common_util.rb | 10 +- .../project/export_file_shared_examples.rb | 50 --- .../project/tree_saver_shared_examples.rb | 272 ------------ 8 files changed, 499 insertions(+), 456 deletions(-) delete mode 100644 ee/spec/support/shared_examples/lib/ee/gitlab/import_export/project/tree_saver_shared_example.rb delete mode 100644 spec/support/shared_examples/lib/gitlab/import_export/project/export_file_shared_examples.rb delete mode 100644 spec/support/shared_examples/lib/gitlab/import_export/project/tree_saver_shared_examples.rb diff --git a/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb b/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb index 74195916c4f673..954b4680b4ec26 100644 --- a/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb +++ b/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb @@ -7,6 +7,7 @@ let_it_be(:group) { create(:group) } let_it_be(:project) { create(:project, group: group) } let_it_be(:issue) { create(:issue, project: project) } + let_it_be(:shared) { project.import_export_shared } let_it_be(:design) { create(:design, :with_file, versions_count: 2, issue: issue) } let_it_be(:note) { create(:diff_note_on_design, noteable: design, project: project, author: user) } @@ -15,16 +16,72 @@ let_it_be(:epic) { create(:epic, group: group) } let_it_be(:epic_issue) { create(:epic_issue, issue: issue, epic: epic) } - let(:shared) { project.import_export_shared } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec_ee" } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } + let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec_ee" } - before do - project.add_maintainer(user) + after :all do + FileUtils.rm_rf(export_path) end - after do - FileUtils.rm_rf(export_path) + shared_examples 'EE saves project tree successfully' do |ndjson_enabled| + include ::ImportExport::CommonUtil + + let_it_be(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } + + let_it_be(:full_path) do + if ndjson_enabled + File.join(shared.export_path, 'tree') + else + File.join(shared.export_path, Gitlab::ImportExport.project_filename) + end + end + + let_it_be(:exportable_path) { 'project' } + + before_all do + Feature.enable(:project_export_as_ndjson) if ndjson_enabled + project.add_maintainer(user) + expect(project_tree_saver.save).to be true + end + + let_it_be(:issue_json) { get_json(full_path, exportable_path, :issues, ndjson_enabled).first } + + describe 'the designs json' do + it 'saves issue.designs correctly' do + expect(issue_json['designs'].size).to eq(1) + end + + it 'saves issue.design_versions correctly' do + actions = issue_json['design_versions'].flat_map { |v| v['actions'] } + + expect(issue_json['design_versions'].size).to eq(2) + issue_json['design_versions'].each do |version| + expect(version['author_id']).to eq(issue.author_id) + end + expect(actions.size).to eq(2) + actions.each do |action| + expect(action['design']).to be_present + end + end + end + + context 'epics' do + it 'has epic_issue' do + expect(issue_json['epic_issue']).not_to be_empty + expect(issue_json['epic_issue']['id']).to eql(epic_issue.id) + end + + it 'has epic' do + expect(issue_json['epic_issue']['epic']['title']).to eql(epic.title) + end + + it 'does not have epic_id' do + expect(issue_json['epic_issue']['epic_id']).to be_nil + end + + it 'does not have issue_id' do + expect(issue_json['epic_issue']['issue_id']).to be_nil + end + end end context 'with JSON' do diff --git a/ee/spec/support/shared_examples/lib/ee/gitlab/import_export/project/tree_saver_shared_example.rb b/ee/spec/support/shared_examples/lib/ee/gitlab/import_export/project/tree_saver_shared_example.rb deleted file mode 100644 index ac55bc53fc0c0e..00000000000000 --- a/ee/spec/support/shared_examples/lib/ee/gitlab/import_export/project/tree_saver_shared_example.rb +++ /dev/null @@ -1,65 +0,0 @@ -# frozen_string_literal: true - -RSpec.shared_examples 'EE saves project tree successfully' do |ndjson_enabled| - include ::ImportExport::CommonUtil - - let(:full_path) do - project_tree_saver.save - - if ndjson_enabled == true - File.join(shared.export_path, 'tree') - else - File.join(shared.export_path, Gitlab::ImportExport.project_filename) - end - end - - let(:exportable_path) { 'project' } - - before do - stub_feature_flags(project_export_as_ndjson: ndjson_enabled) - end - - it 'saves successfully' do - expect(project_tree_saver.save).to be true - end - - describe 'the designs json' do - let(:issue_json) { saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first } - - it 'saves issue.designs correctly' do - expect(issue_json['designs'].size).to eq(1) - end - - it 'saves issue.design_versions correctly' do - actions = issue_json['design_versions'].flat_map { |v| v['actions'] } - - expect(issue_json['design_versions'].size).to eq(2) - issue_json['design_versions'].each do |version| - expect(version['author_id']).to eq(issue.author_id) - end - expect(actions.size).to eq(2) - actions.each do |action| - expect(action['design']).to be_present - end - end - end - - context 'epics' do - it 'has epic_issue' do - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']).not_to be_empty - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']['id']).to eql(epic_issue.id) - end - - it 'has epic' do - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']['epic']['title']).to eql(epic.title) - end - - it 'does not have epic_id' do - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']['epic_id']).to be_nil - end - - it 'does not have issue_id' do - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['epic_issue']['issue_id']).to be_nil - end - end -end diff --git a/spec/features/projects/import_export/export_file_spec.rb b/spec/features/projects/import_export/export_file_spec.rb index 6c560a341cee0d..1e13afc6033b97 100644 --- a/spec/features/projects/import_export/export_file_spec.rb +++ b/spec/features/projects/import_export/export_file_spec.rb @@ -38,6 +38,27 @@ sign_in(user) end + shared_examples 'export file without sensitive words' do + it 'exports a project successfully', :sidekiq_inline do + export_project_and_download_file(page, project) + + in_directory_with_expanded_export(project) do |exit_status, tmpdir| + expect(exit_status).to eq(0) + + project_json_path = File.join(tmpdir, 'project.json') + expect(File).to exist(project_json_path) + + project_hash = JSON.parse(IO.read(project_json_path)) + + sensitive_words.each do |sensitive_word| + found = find_sensitive_attributes(sensitive_word, project_hash) + + expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) + end + end + end + end + context "with legacy export" do before do stub_feature_flags(streaming_serializer: false) @@ -52,6 +73,7 @@ stub_feature_flags(streaming_serializer: true) stub_feature_flags(project_export_as_ndjson: false) end + it_behaves_like "export file without sensitive words" end @@ -61,19 +83,8 @@ stub_feature_flags(project_export_as_ndjson: true) end - it 'exports a project successfully', :sidekiq_might_not_need_inline do - visit edit_project_path(project) - - expect(page).to have_content('Export project') - - find(:link, 'Export project').send_keys(:return) - - visit edit_project_path(project) - - expect(page).to have_content('Download export') - - expect(project.export_status).to eq(:finished) - expect(project.export_file.path).to include('tar.gz') + it 'exports a project successfully', :sidekiq_inline do + export_project_and_download_file(page, project) in_directory_with_expanded_export(project) do |exit_status, tmpdir| expect(exit_status).to eq(0) @@ -99,23 +110,37 @@ end end end + end + end - def failure_message(key_found, parent, sensitive_word) - <<-MSG - Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} + def export_project_and_download_file(page, project) + visit edit_project_path(project) - If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. + expect(page).to have_content('Export project') - Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the - correspondent hash or model as the value. + find(:link, 'Export project').send_keys(:return) - Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be - reset (to prevent duplicate column problems while importing to the same instance). + visit edit_project_path(project) - IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} - CURRENT_SPEC: #{__FILE__} - MSG - end - end + expect(page).to have_content('Download export') + expect(project.export_status).to eq(:finished) + expect(project.export_file.path).to include('tar.gz') + end + + def failure_message(key_found, parent, sensitive_word) + <<-MSG + Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} + + If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. + + Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the + correspondent hash or model as the value. + + Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be + reset (to prevent duplicate column problems while importing to the same instance). + + IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} + CURRENT_SPEC: #{__FILE__} + MSG end end diff --git a/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb index 1b14141ddbae0b..bae3672474cb65 100644 --- a/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb +++ b/spec/lib/gitlab/import_export/json/ndjson_writer_spec.rb @@ -5,7 +5,7 @@ describe Gitlab::ImportExport::JSON::NdjsonWriter do include ImportExport::CommonUtil - let(:path) { "#{Dir.tmpdir}/legacy_writer_spec/tree" } + let(:path) { "#{Dir.tmpdir}/ndjson_writer_spec/tree" } let(:exportable_path) { 'projects' } subject { described_class.new(path) } diff --git a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb index d1da7236824ef7..d859af5df02121 100644 --- a/spec/lib/gitlab/import_export/project/tree_saver_spec.rb +++ b/spec/lib/gitlab/import_export/project/tree_saver_spec.rb @@ -3,21 +3,374 @@ require 'spec_helper' describe Gitlab::ImportExport::Project::TreeSaver do - let(:shared) { project.import_export_shared } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } - let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } - let(:user) { create(:user) } - let!(:project) { setup_project } - - before do - project.add_maintainer(user) - allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path) - allow_any_instance_of(MergeRequest).to receive(:source_branch_sha).and_return('ABCD') - allow_any_instance_of(MergeRequest).to receive(:target_branch_sha).and_return('DCBA') - end + let_it_be(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" } + let_it_be(:exportable_path) { 'project' } + + shared_examples 'saves project tree successfully' do |ndjson_enabled| + include ImportExport::CommonUtil + + subject { get_json(full_path, exportable_path, relation_name, ndjson_enabled) } + + describe 'saves project tree attributes' do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let_it_be(:project) { setup_project } + let_it_be(:shared) { project.import_export_shared } + let_it_be(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared) } + + let(:relation_name) { :projects } + + let_it_be(:full_path) do + if ndjson_enabled + File.join(shared.export_path, 'tree') + else + File.join(shared.export_path, Gitlab::ImportExport.project_filename) + end + end + + before_all do + Feature.enable(:project_export_as_ndjson) if ndjson_enabled + project.add_maintainer(user) + project_tree_saver.save + end + + after :all do + FileUtils.rm_rf(export_path) + end + + context 'with project root' do + it { is_expected.to include({ 'description' => 'description', 'visibility_level' => 20 }) } + + it { is_expected.not_to include("runners_token" => 'token') } + + it 'has approvals_before_merge set' do + expect(subject['approvals_before_merge']).to eq(1) + end + end + + context 'with milestones' do + let(:relation_name) { :milestones } + + it { is_expected.not_to be_empty } + end + + context 'with merge_requests' do + let(:relation_name) { :merge_requests } + + it { is_expected.not_to be_empty } + + it 'has merge request\'s milestones' do + expect(subject.first['milestone']).not_to be_empty + end + it 'has merge request\'s source branch SHA' do + expect(subject.first['source_branch_sha']).to eq('b83d6e391c22777fca1ed3012fce84f633d7fed0') + end + + it 'has merge request\'s target branch SHA' do + expect(subject.first['target_branch_sha']).to eq('0b4bc9a49b562e85de7cc9e834518ea6828729b9') + end + + it 'has events' do + expect(subject.first['milestone']['events']).not_to be_empty + end + + it 'has merge requests diffs' do + expect(subject.first['merge_request_diff']).not_to be_empty + end + + it 'has merge request diff files' do + expect(subject.first['merge_request_diff']['merge_request_diff_files']).not_to be_empty + end + + it 'has merge request diff commits' do + expect(subject.first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty + end + + it 'has merge requests comments' do + expect(subject.first['notes']).not_to be_empty + end + + it 'has author on merge requests comments' do + expect(subject.first['notes'].first['author']).not_to be_empty + end + + it 'has merge request resource label events' do + expect(subject.first['resource_label_events']).not_to be_empty + end + end + + context 'with snippets' do + let(:relation_name) { :snippets } + + it { is_expected.not_to be_empty } + + it 'has snippet notes' do + expect(subject.first['notes']).not_to be_empty + end + end + + context 'with releases' do + let(:relation_name) { :releases } + + it { is_expected.not_to be_empty } + + it 'has no author on releases' do + expect(subject.first['author']).to be_nil + end + + it 'has the author ID on releases' do + expect(subject.first['author_id']).not_to be_nil + end + end + + context 'with issues' do + let(:relation_name) { :issues } + + it { is_expected.not_to be_empty } + + it 'has issue comments' do + notes = subject.first['notes'] + + expect(notes).not_to be_empty + expect(notes.first['type']).to eq('DiscussionNote') + end + + it 'has issue assignees' do + expect(subject.first['issue_assignees']).not_to be_empty + end + + it 'has author on issue comments' do + expect(subject.first['notes'].first['author']).not_to be_empty + end + + it 'has labels associated to records' do + expect(subject.first['label_links'].first['label']).not_to be_empty + end + + it 'has project and group labels' do + label_types = subject.first['label_links'].map { |link| link['label']['type'] } + + expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) + end + + it 'has priorities associated to labels' do + priorities = subject.first['label_links'].flat_map { |link| link['label']['priorities'] } + + expect(priorities).not_to be_empty + end + + it 'has issue resource label events' do + expect(subject.first['resource_label_events']).not_to be_empty + end + end + + context 'with ci_pipelines' do + let(:relation_name) { :ci_pipelines } + + it { is_expected.not_to be_empty } + + it 'has pipeline stages' do + expect(subject.dig(0, 'stages')).not_to be_empty + end + + it 'has pipeline statuses' do + expect(subject.dig(0, 'stages', 0, 'statuses')).not_to be_empty + end + + it 'has pipeline builds' do + builds_count = subject.dig(0, 'stages', 0, 'statuses') + .count { |hash| hash['type'] == 'Ci::Build' } + + expect(builds_count).to eq(1) + end + + it 'has ci pipeline notes' do + expect(subject.first['notes']).not_to be_empty + end + end + + context 'with labels' do + let(:relation_name) { :labels } + + it { is_expected.not_to be_empty } + end + + context 'with services' do + let(:relation_name) { :services } + + it 'saves the correct service type' do + expect(subject.first['type']).to eq('CustomIssueTrackerService') + end + + it 'saves the properties for a service' do + expect(subject.first['properties']).to eq('one' => 'value') + end + end + + context 'with project_feature' do + let(:relation_name) { :project_feature } + + it { is_expected.not_to be_empty } + + it 'has project feature' do + expect(subject["issues_access_level"]).to eq(ProjectFeature::DISABLED) + expect(subject["wiki_access_level"]).to eq(ProjectFeature::ENABLED) + expect(subject["builds_access_level"]).to eq(ProjectFeature::PRIVATE) + end + end + + context 'with custom_attributes' do + let(:relation_name) { :custom_attributes } + + it 'has custom attributes' do + expect(subject.count).to eq(2) + end + end + + context 'with badges' do + let(:relation_name) { :custom_attributes } + + it 'has badges' do + expect(subject.count).to eq(2) + end + end + + context 'with project_members' do + let(:relation_name) { :project_members } + + it { is_expected.not_to be_empty } + end + + context 'with boards' do + let(:relation_name) { :boards } + + it { is_expected.not_to be_empty } + end + end + + describe '#saves project tree' do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let(:project) { setup_project } + let(:full_path) do + if ndjson_enabled + File.join(shared.export_path, 'tree') + else + File.join(shared.export_path, Gitlab::ImportExport.project_filename) + end + end + let(:shared) { project.import_export_shared } + let(:params) { {} } + + let(:project_tree_saver ) { described_class.new(project: project, current_user: user, shared: shared, params: params) } + + before do + stub_feature_flags(project_export_as_ndjson: ndjson_enabled) + project.add_maintainer(user) + + FileUtils.rm_rf(export_path) + end + + after do + FileUtils.rm_rf(export_path) + end + + context 'overrides group members' do + let(:user2) { create(:user, email: 'group@member.com') } + let(:relation_name) { :project_members } + + let(:member_emails) do + emails = subject.map do |pm| + pm['user']['email'] + end + emails + end + + before do + group.add_developer(user2) + end + + context 'when has no permission' do + before do + group.add_developer(user) + project_tree_saver.save + end + + it 'does not export group members' do + expect(member_emails).not_to include('group@member.com') + end + end + + context 'when has permission as maintainer' do + before do + group.add_maintainer(user) + + project_tree_saver.save + end + + it 'does not export group members' do + expect(member_emails).not_to include('group@member.com') + end + end + + context 'when has permission as group owner' do + before do + group.add_owner(user) + + project_tree_saver.save + end + + it 'exports group members as group owner' do + expect(member_emails).to include('group@member.com') + end + end + + context 'as admin' do + let(:user) { create(:admin) } + + before do + project_tree_saver.save + end + + it 'exports group members as admin' do + expect(member_emails).to include('group@member.com') + end + + it 'exports group members as project members' do + member_types = subject.map { |pm| pm['source_type'] } + + expect(member_types).to all(eq('Project')) + end + end + end + + context 'with description override' do + let(:params) { { description: 'Foo Bar' } } + let(:relation_name) { :projects } + + before do + project_tree_saver.save + end + + it { is_expected.to include({ 'description' => params[:description] }) } + end + + it 'saves project successfully' do + expect(project_tree_saver.save).to be true + end + + it 'does not complain about non UTF-8 characters in MR diff files' do + ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") + + expect(project_tree_saver.save).to be true + end + + it 'has no when YML attributes but only the DB column' do + expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) - after do - FileUtils.rm_rf(export_path) + project_tree_saver.save + end + end end context 'with JSON' do @@ -30,20 +383,17 @@ def setup_project release = create(:release) - group = create(:group) project = create(:project, - :public, - :repository, - :issues_disabled, - :wiki_enabled, - :builds_private, - description: 'description', - releases: [release], - group: group, - approvals_before_merge: 1 - ) - allow(project).to receive(:commit).and_return(Commit.new(RepoHelpers.sample_commit, project)) + :public, + :repository, + :issues_disabled, + :wiki_enabled, + :builds_private, + description: 'description', + releases: [release], + group: group, + approvals_before_merge: 1) issue = create(:issue, assignees: [user], project: project) snippet = create(:project_snippet, project: project) @@ -64,9 +414,9 @@ def setup_project mr_note = create(:note, noteable: merge_request, project: project) create(:note, noteable: snippet, project: project) create(:note_on_commit, - author: user, - project: project, - commit_id: ci_build.pipeline.sha) + author: user, + project: project, + commit_id: ci_build.pipeline.sha) create(:system_note_metadata, action: 'description', note: discussion_note) create(:system_note_metadata, commit_count: 1, action: 'commit', note: mr_note) diff --git a/spec/support/import_export/common_util.rb b/spec/support/import_export/common_util.rb index da5a537911263d..efe14b7244cb40 100644 --- a/spec/support/import_export/common_util.rb +++ b/spec/support/import_export/common_util.rb @@ -26,15 +26,13 @@ def test_tmp_path "tmp/tests/gitlab-test/import_export" end - def saved_relations(path, exportable_path, key, ndjson_enabled) - if ndjson_enabled == true + def get_json(path, exportable_path, key, ndjson_enabled) + if ndjson_enabled json = if key == :projects consume_attributes(path, exportable_path) else consume_relations(path, exportable_path, key) end - - json = json.first if key == :project_feature else json = project_json(path) json = json[key.to_s] unless key == :projects @@ -86,11 +84,11 @@ def consume_relations(dir_path, exportable_path, key) relations << json end - relations.flatten + key == :project_feature ? relations.first : relations.flatten end def project_json(filename) - ::JSON.parse(IO.read(filename)) + ActiveSupport::JSON.decode(IO.read(filename)) end end end diff --git a/spec/support/shared_examples/lib/gitlab/import_export/project/export_file_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/project/export_file_shared_examples.rb deleted file mode 100644 index 649967ed56a51e..00000000000000 --- a/spec/support/shared_examples/lib/gitlab/import_export/project/export_file_shared_examples.rb +++ /dev/null @@ -1,50 +0,0 @@ -# frozen_string_literal: true - -RSpec.shared_examples 'export file without sensitive words' do - it 'exports a project successfully', :sidekiq_might_not_need_inline do - visit edit_project_path(project) - - expect(page).to have_content('Export project') - - find(:link, 'Export project').send_keys(:return) - - visit edit_project_path(project) - - expect(page).to have_content('Download export') - - expect(project.export_status).to eq(:finished) - expect(project.export_file.path).to include('tar.gz') - - in_directory_with_expanded_export(project) do |exit_status, tmpdir| - expect(exit_status).to eq(0) - - project_json_path = File.join(tmpdir, 'project.json') - expect(File).to exist(project_json_path) - - project_hash = JSON.parse(IO.read(project_json_path)) - - sensitive_words.each do |sensitive_word| - found = find_sensitive_attributes(sensitive_word, project_hash) - - expect(found).to be_nil, failure_message(found.try(:key_found), found.try(:parent), sensitive_word) - end - end - end - - def failure_message(key_found, parent, sensitive_word) - <<-MSG - Found a new sensitive word <#{key_found}>, which is part of the hash #{parent.inspect} - - If you think this information shouldn't get exported, please exclude the model or attribute in IMPORT_EXPORT_CONFIG. - - Otherwise, please add the exception to +safe_list+ in CURRENT_SPEC using #{sensitive_word} as the key and the - correspondent hash or model as the value. - - Also, if the attribute is a generated unique token, please add it to RelationFactory::TOKEN_RESET_MODELS if it needs to be - reset (to prevent duplicate column problems while importing to the same instance). - - IMPORT_EXPORT_CONFIG: #{Gitlab::ImportExport.config_file} - CURRENT_SPEC: #{__FILE__} - MSG - end -end diff --git a/spec/support/shared_examples/lib/gitlab/import_export/project/tree_saver_shared_examples.rb b/spec/support/shared_examples/lib/gitlab/import_export/project/tree_saver_shared_examples.rb deleted file mode 100644 index 5fd1bd9baf9ce7..00000000000000 --- a/spec/support/shared_examples/lib/gitlab/import_export/project/tree_saver_shared_examples.rb +++ /dev/null @@ -1,272 +0,0 @@ -# frozen_string_literal: true - -RSpec.shared_examples 'saves project tree successfully' do |ndjson_enabled| - include ImportExport::CommonUtil - - let(:full_path) do - project_tree_saver.save - - if ndjson_enabled == true - File.join(shared.export_path, 'tree') - else - File.join(shared.export_path, Gitlab::ImportExport.project_filename) - end - end - - let(:exportable_path) { 'project' } - - before do - stub_feature_flags(project_export_as_ndjson: ndjson_enabled) - end - - it 'saves project successfully' do - expect(project_tree_saver.save).to be true - end - # It is not duplicated indoes not contain the runners token - # `spec/lib/gitlab/import_export/fast_hash_serializer_spec.rb` - context 'with description override' do - let(:params) { { description: 'Foo Bar' } } - let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared, params: params) } - - it 'overrides the project description' do - expect(saved_relations(full_path, exportable_path, :projects, ndjson_enabled)).to include({ 'description' => params[:description] }) - end - end - - it 'saves the correct json' do - expect(saved_relations(full_path, exportable_path, :projects, ndjson_enabled)).to include({ 'description' => 'description', 'visibility_level' => 20 }) - end - - it 'has approvals_before_merge set' do - expect(saved_relations(full_path, exportable_path, :projects, ndjson_enabled)['approvals_before_merge']).to eq(1) - end - - it 'has milestones' do - expect(saved_relations(full_path, exportable_path, :milestones, ndjson_enabled)).not_to be_empty - end - - it 'has merge requests' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled)).not_to be_empty - end - - it 'has merge request\'s milestones' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['milestone']).not_to be_empty - end - it 'has merge request\'s source branch SHA' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['source_branch_sha']).to eq('ABCD') - end - - it 'has merge request\'s target branch SHA' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['target_branch_sha']).to eq('DCBA') - end - - it 'has events' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['milestone']['events']).not_to be_empty - end - - it 'has snippets' do - expect(saved_relations(full_path, exportable_path, :snippets, ndjson_enabled)).not_to be_empty - end - - it 'has snippet notes' do - expect(saved_relations(full_path, exportable_path, :snippets, ndjson_enabled).first['notes']).not_to be_empty - end - - it 'has releases' do - expect(saved_relations(full_path, exportable_path, :releases, ndjson_enabled)).not_to be_empty - end - - it 'has no author on releases' do - expect(saved_relations(full_path, exportable_path, :releases, ndjson_enabled).first['author']).to be_nil - end - - it 'has the author ID on releases' do - expect(saved_relations(full_path, exportable_path, :releases, ndjson_enabled).first['author_id']).not_to be_nil - end - - it 'has issues' do - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled)).not_to be_empty - end - - it 'has issue comments' do - notes = saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['notes'] - - expect(notes).not_to be_empty - expect(notes.first['type']).to eq('DiscussionNote') - end - - it 'has issue assignees' do - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['issue_assignees']).not_to be_empty - end - - it 'has author on issue comments' do - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['notes'].first['author']).not_to be_empty - end - - it 'has project members' do - expect(saved_relations(full_path, exportable_path, :project_members, ndjson_enabled)).not_to be_empty - end - - it 'has merge requests diffs' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['merge_request_diff']).not_to be_empty - end - - it 'has merge request diff files' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['merge_request_diff']['merge_request_diff_files']).not_to be_empty - end - - it 'has merge request diff commits' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['merge_request_diff']['merge_request_diff_commits']).not_to be_empty - end - - it 'has merge requests comments' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['notes']).not_to be_empty - end - - it 'has author on merge requests comments' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['notes'].first['author']).not_to be_empty - end - - it 'has pipeline stages' do - expect(saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled).dig(0, 'stages')).not_to be_empty - end - - it 'has pipeline statuses' do - expect(saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled).dig(0, 'stages', 0, 'statuses')).not_to be_empty - end - - it 'has pipeline builds' do - builds_count = saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled).dig(0, 'stages', 0, 'statuses') - .count { |hash| hash['type'] == 'Ci::Build' } - - expect(builds_count).to eq(1) - end - - it 'has no when YML attributes but only the DB column' do - expect_any_instance_of(Gitlab::Ci::YamlProcessor).not_to receive(:build_attributes) - - project_tree_saver.save - end - - it 'has pipeline commits' do - expect(saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled)).not_to be_empty - end - - it 'has ci pipeline notes' do - expect(saved_relations(full_path, exportable_path, :ci_pipelines, ndjson_enabled).first['notes']).not_to be_empty - end - - it 'has labels with no associations' do - expect(saved_relations(full_path, exportable_path, :labels, ndjson_enabled)).not_to be_empty - end - - it 'has labels associated to records' do - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['label_links'].first['label']).not_to be_empty - end - - it 'has project and group labels' do - label_types = saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['label_links'].map { |link| link['label']['type'] } - - expect(label_types).to match_array(%w(ProjectLabel GroupLabel)) - end - - it 'has priorities associated to labels' do - priorities = saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['label_links'].flat_map { |link| link['label']['priorities'] } - - expect(priorities).not_to be_empty - end - - it 'has issue resource label events' do - expect(saved_relations(full_path, exportable_path, :issues, ndjson_enabled).first['resource_label_events']).not_to be_empty - end - - it 'has merge request resource label events' do - expect(saved_relations(full_path, exportable_path, :merge_requests, ndjson_enabled).first['resource_label_events']).not_to be_empty - end - - it 'saves the correct service type' do - expect(saved_relations(full_path, exportable_path, :services, ndjson_enabled).first['type']).to eq('CustomIssueTrackerService') - end - - it 'saves the properties for a service' do - expect(saved_relations(full_path, exportable_path, :services, ndjson_enabled).first['properties']).to eq('one' => 'value') - end - - it 'has project feature' do - project_feature = saved_relations(full_path, exportable_path, :project_feature, ndjson_enabled) - expect(project_feature).not_to be_empty - expect(project_feature["issues_access_level"]).to eq(ProjectFeature::DISABLED) - expect(project_feature["wiki_access_level"]).to eq(ProjectFeature::ENABLED) - expect(project_feature["builds_access_level"]).to eq(ProjectFeature::PRIVATE) - end - - it 'has custom attributes' do - expect(saved_relations(full_path, exportable_path, :custom_attributes, ndjson_enabled).count).to eq(2) - end - - it 'has badges' do - expect(saved_relations(full_path, exportable_path, :project_badges, ndjson_enabled).count).to eq(2) - end - - it 'does not complain about non UTF-8 characters in MR diff files' do - ActiveRecord::Base.connection.execute("UPDATE merge_request_diff_files SET diff = '---\n- :diff: !binary |-\n LS0tIC9kZXYvbnVsbAorKysgYi9pbWFnZXMvbnVjb3IucGRmCkBAIC0wLDAg\n KzEsMTY3OSBAQAorJVBERi0xLjUNJeLjz9MNCisxIDAgb2JqDTw8L01ldGFk\n YXR'") - - expect(project_tree_saver.save).to be true - end - - context 'group members' do - let(:user2) { create(:user, email: 'group@member.com') } - let(:member_emails) do - emails = saved_relations(full_path, exportable_path, :project_members, ndjson_enabled).map do |pm| - pm['user']['email'] - end - emails - end - - before do - Group.first.add_developer(user2) - end - - it 'does not export group members if it has no permission' do - Group.first.add_developer(user) - - expect(member_emails).not_to include('group@member.com') - end - - it 'does not export group members as maintainer' do - Group.first.add_maintainer(user) - - expect(member_emails).not_to include('group@member.com') - end - - it 'exports group members as group owner' do - Group.first.add_owner(user) - - expect(member_emails).to include('group@member.com') - end - - context 'as admin' do - let(:user) { create(:admin) } - - it 'exports group members as admin' do - expect(member_emails).to include('group@member.com') - end - - it 'exports group members as project members' do - member_types = saved_relations(full_path, exportable_path, :project_members, ndjson_enabled).map { |pm| pm['source_type'] } - - expect(member_types).to all(eq('Project')) - end - end - end - - context 'project attributes' do - it 'does not contain the runners token' do - expect(saved_relations(full_path, exportable_path, :projects, ndjson_enabled)).not_to include("runners_token" => 'token') - end - end - - it 'has a board and a list' do - expect(saved_relations(full_path, exportable_path, :boards, ndjson_enabled).first['lists']).not_to be_empty - end -end -- GitLab