diff --git a/app/services/bulk_imports/relation_export_service.rb b/app/services/bulk_imports/relation_export_service.rb index c43f0d8cb4fa389c7340b62615598f156f57a992..b1efa8811800b3dc75dc392257f4258c136847de 100644 --- a/app/services/bulk_imports/relation_export_service.rb +++ b/app/services/bulk_imports/relation_export_service.rb @@ -65,7 +65,7 @@ def remove_existing_export_file!(export) def export_service @export_service ||= if config.tree_relation?(relation) || config.self_relation?(relation) - TreeExportService.new(portable, config.export_path, relation) + TreeExportService.new(portable, config.export_path, relation, user) elsif config.file_relation?(relation) FileExportService.new(portable, config.export_path, relation) else diff --git a/app/services/bulk_imports/tree_export_service.rb b/app/services/bulk_imports/tree_export_service.rb index 8e885e590d1b7eac98f46ad9e104fa466427b9a3..b6f094da5585dfc900427806a412c8736ed0214e 100644 --- a/app/services/bulk_imports/tree_export_service.rb +++ b/app/services/bulk_imports/tree_export_service.rb @@ -2,11 +2,12 @@ module BulkImports class TreeExportService - def initialize(portable, export_path, relation) + def initialize(portable, export_path, relation, user) @portable = portable @export_path = export_path @relation = relation @config = FileTransfer.config_for(portable) + @user = user end def execute @@ -27,7 +28,7 @@ def exported_filename private - attr_reader :export_path, :portable, :relation, :config + attr_reader :export_path, :portable, :relation, :config, :user # rubocop: disable CodeReuse/Serializer def serializer @@ -35,7 +36,8 @@ def serializer portable, config.portable_tree, json_writer, - exportable_path: '' + exportable_path: '', + current_user: user ) end # rubocop: enable CodeReuse/Serializer diff --git a/doc/development/import_export.md b/doc/development/import_export.md index 6cbbb6bf7167326efc0c1ae2755f9df3185d1659..c66ac0418ac63288b94484dc8e42ac90f16db75b 100644 --- a/doc/development/import_export.md +++ b/doc/development/import_export.md @@ -305,6 +305,29 @@ export_reorders: nulls_position: :nulls_last ``` +### Conditional export + +When associated resources are from outside the project, you might need to +validate that a user who is exporting the project or group can access these +associations. `include_if_exportable` accepts an array of associations for a +resource. During export, the `exportable_association?` method on the resource +is called with the association's name and user to validate if associated +resource can be included in the export. + +For example: + +```yaml +include_if_exportable: + project: + issues: + - epic_issue +``` + +This definition: + +1. Calls the issue's `exportable_association?(:epic_issue, current_user: current_user)` method. +1. If the method returns true, includes the issue's `epic_issue` association for the issue. + ### Import The import job status moves from `none` to `finished` or `failed` into different states: diff --git a/ee/app/models/ee/issue.rb b/ee/app/models/ee/issue.rb index 31b1c871e2a06012fc27a6d8a2dc78fd128c9eaf..56148ae617798dea89d82ae63d83ee705798d416 100644 --- a/ee/app/models/ee/issue.rb +++ b/ee/app/models/ee/issue.rb @@ -323,6 +323,12 @@ def has_epic? epic_issue.present? end + def exportable_association?(key, current_user: nil) + return false unless key == :epic_issue + + epic.present? && current_user&.can?(:read_epic, epic) + end + private def blocking_issues_ids diff --git a/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb b/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb index d354e9e2fc2121d3a5b48c50cdb26d63881d33d9..72ff98ab790a2f419b5c78c168895677856d8600 100644 --- a/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb +++ b/ee/spec/lib/ee/gitlab/import_export/project/tree_saver_spec.rb @@ -7,7 +7,6 @@ let_it_be(:group) { create(:group) } let_it_be(:project) { create(:project, group: group) } let_it_be(:issue) { create(:issue, project: project) } - let_it_be(:shared) { project.import_export_shared } let_it_be(:note2) { create(:note, noteable: issue, project: project, author: user) } let_it_be(:epic) { create(:epic, group: group) } @@ -17,16 +16,10 @@ let_it_be(:push_rule) { create(:push_rule, project: project, max_file_size: 10) } - after :all do - FileUtils.rm_rf(export_path) - end - shared_examples 'EE saves project tree successfully' do |ndjson_enabled| include ::ImportExport::CommonUtil - let_it_be(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } - - let_it_be(:full_path) do + let(:full_path) do if ndjson_enabled File.join(shared.export_path, 'tree') else @@ -34,36 +27,41 @@ end end - let_it_be(:exportable_path) { 'project' } - - before_all do - RSpec::Mocks.with_temporary_scope do - stub_all_feature_flags - stub_feature_flags(project_export_as_ndjson: ndjson_enabled) - project.add_maintainer(user) - - expect(project_tree_saver.save).to be true - end + let(:shared) { project.import_export_shared } + let(:project_tree_saver) { described_class.new(project: project, current_user: user, shared: shared) } + let(:issue_json) { get_json(full_path, exportable_path, :issues, ndjson_enabled).first } + let(:exportable_path) { 'project' } + let(:epics_available) { true } + + before do + stub_all_feature_flags + stub_feature_flags(project_export_as_ndjson: ndjson_enabled) + stub_licensed_features(epics: epics_available) + project.add_maintainer(user) end - let_it_be(:issue_json) { get_json(full_path, exportable_path, :issues, ndjson_enabled).first } + after do + FileUtils.rm_rf(export_path) + FileUtils.rm_rf(full_path) + end context 'epics' do - it 'has epic_issue' do + it 'contains issue epic object', :aggregate_failures do + expect(project_tree_saver.save).to be true expect(issue_json['epic_issue']).not_to be_empty expect(issue_json['epic_issue']['id']).to eql(epic_issue.id) - end - - it 'has epic' do expect(issue_json['epic_issue']['epic']['title']).to eql(epic.title) - end - - it 'does not have epic_id' do expect(issue_json['epic_issue']['epic_id']).to be_nil + expect(issue_json['epic_issue']['issue_id']).to be_nil end - it 'does not have issue_id' do - expect(issue_json['epic_issue']['issue_id']).to be_nil + context 'when epic is not readable' do + let(:epics_available) { false } + + it 'filters out inaccessible epic object' do + expect(project_tree_saver.save).to be true + expect(issue_json['epic_issue']).to be_nil + end end end @@ -74,6 +72,7 @@ end it 'has security settings' do + expect(project_tree_saver.save).to be true expect(security_json['auto_fix_dependency_scanning']).to be_truthy end end @@ -85,6 +84,7 @@ end it 'has push rules' do + expect(project_tree_saver.save).to be true expect(push_rule_json['max_file_size']).to eq(10) expect(push_rule_json['force_push_regex']).to eq('feature\/.*') end diff --git a/ee/spec/models/issue_spec.rb b/ee/spec/models/issue_spec.rb index 45f48e23c04622146c4f56c928d33d7283a2a5bd..114570778f940e1b4a2b59168dec1d6e7625ecaf 100644 --- a/ee/spec/models/issue_spec.rb +++ b/ee/spec/models/issue_spec.rb @@ -1297,4 +1297,47 @@ it { is_expected.to eq true } end end + + describe '#exportable_association?' do + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group, :private) } + let_it_be(:project) { create(:project, group: group) } + let_it_be_with_reload(:issue) { create(:issue, project: project) } + let_it_be(:epic) { create(:epic, group: group) } + let_it_be(:epic_issue) { create(:epic_issue, issue: issue, epic: epic) } + + let(:key) { :epic_issue } + + subject { issue.exportable_association?(key, current_user: user) } + + it { is_expected.to be_falsey } + + context 'when epics are available' do + before do + stub_licensed_features(epics: true) + end + + it { is_expected.to be_falsey } + + context 'when user can read epic' do + before do + group.add_developer(user) + end + + it { is_expected.to be_truthy } + + context 'for an unknown key' do + let(:key) { :labels } + + it { is_expected.to be_falsey } + end + + context 'for an unauthenticated user' do + let(:user) { nil } + + it { is_expected.to be_falsey } + end + end + end + end end diff --git a/lib/gitlab/import_export/attributes_finder.rb b/lib/gitlab/import_export/attributes_finder.rb index 4abc3da11909b1427bd9d8803deb02aeb42ff684..8843b4f5755655092065ab466b7e0ca1161dcd29 100644 --- a/lib/gitlab/import_export/attributes_finder.rb +++ b/lib/gitlab/import_export/attributes_finder.rb @@ -12,6 +12,7 @@ def initialize(config:) @methods = config[:methods] || {} @preloads = config[:preloads] || {} @export_reorders = config[:export_reorders] || {} + @include_if_exportable = config[:include_if_exportable] || {} end def find_root(model_key) @@ -35,7 +36,8 @@ def find(model_key, model_tree) methods: @methods[model_key], include: resolve_model_tree(model_tree), preload: resolve_preloads(model_key, model_tree), - export_reorder: @export_reorders[model_key] + export_reorder: @export_reorders[model_key], + include_if_exportable: @include_if_exportable[model_key] }.compact end diff --git a/lib/gitlab/import_export/group/tree_saver.rb b/lib/gitlab/import_export/group/tree_saver.rb index 796b9258e57b0ddc6a11ebd0ffa28236de345df9..b4c86c3fc7f046a3caebbf3cc1d2ca9c6a0cf9f0 100644 --- a/lib/gitlab/import_export/group/tree_saver.rb +++ b/lib/gitlab/import_export/group/tree_saver.rb @@ -46,7 +46,8 @@ def serialize(group) group, group_tree, json_writer, - exportable_path: "groups/#{group.id}" + exportable_path: "groups/#{group.id}", + current_user: @current_user ).execute end diff --git a/lib/gitlab/import_export/json/streaming_serializer.rb b/lib/gitlab/import_export/json/streaming_serializer.rb index 42e014e91f887786828270919a12f1fac33fdf75..99396d64779e5f5cb8abb496861e03385a41d0ac 100644 --- a/lib/gitlab/import_export/json/streaming_serializer.rb +++ b/lib/gitlab/import_export/json/streaming_serializer.rb @@ -14,8 +14,9 @@ def to_json(*_args) end end - def initialize(exportable, relations_schema, json_writer, exportable_path:, logger: Gitlab::Export::Logger) + def initialize(exportable, relations_schema, json_writer, current_user:, exportable_path:, logger: Gitlab::Export::Logger) @exportable = exportable + @current_user = current_user @exportable_path = exportable_path @relations_schema = relations_schema @json_writer = json_writer @@ -59,7 +60,7 @@ def serialize_relation(definition) private - attr_reader :json_writer, :relations_schema, :exportable, :logger + attr_reader :json_writer, :relations_schema, :exportable, :logger, :current_user def serialize_many_relations(key, records, options) log_relation_export(key, records.size) @@ -73,7 +74,7 @@ def serialize_many_relations(key, records, options) batch.each do |record| before_read_callback(record) - items << Raw.new(record.to_json(options)) + items << exportable_json_record(record, options, key) after_read_callback(record) end @@ -83,6 +84,27 @@ def serialize_many_relations(key, records, options) json_writer.write_relation_array(@exportable_path, key, enumerator) end + def exportable_json_record(record, options, key) + associations = relations_schema[:include_if_exportable]&.dig(key) + return Raw.new(record.to_json(options)) unless associations && options[:include] + + filtered_options = options.deep_dup + associations.each do |association| + filtered_options[:include].delete_if do |option| + !exportable_json_association?(option, record, association.to_sym) + end + end + + Raw.new(record.to_json(filtered_options)) + end + + def exportable_json_association?(option, record, association) + return true unless option.has_key?(association) + return false unless record.respond_to?(:exportable_association?) + + record.exportable_association?(association, current_user: current_user) + end + def batch(relation, key) opts = { of: BATCH_SIZE } order_by = reorders(relation, key) @@ -111,7 +133,7 @@ def serialize_many_each(key, records, options) enumerator = Enumerator.new do |items| records.each do |record| - items << Raw.new(record.to_json(options)) + items << exportable_json_record(record, options, key) end end @@ -121,7 +143,7 @@ def serialize_many_each(key, records, options) def serialize_single_relation(key, record, options) log_relation_export(key) - json = Raw.new(record.to_json(options)) + json = exportable_json_record(record, options, key) json_writer.write_relation(@exportable_path, key, json) end diff --git a/lib/gitlab/import_export/project/import_export.yml b/lib/gitlab/import_export/project/import_export.yml index 77311e14803368f3e9b558ab50bc5141f413a22e..069eab7b5bf9a115df17d21dc67fa31ca695d6be 100644 --- a/lib/gitlab/import_export/project/import_export.yml +++ b/lib/gitlab/import_export/project/import_export.yml @@ -1155,3 +1155,28 @@ ee: - :user_id - :action - :created_at + + preloads: + issues: + epic: + + # When associated resources are from outside the project, you might need to + # validate that a user who is exporting the project or group can access these + # associations. `include_if_exportable` accepts an array of associations for a + # resource. During export, the `exportable_association?` method on the + # resource is called with the association's name and user to validate if + # associated resource can be included in the export. + # + # This definition will call issue's `exportable_association?(:epic_issue, + # current_user: current_user)` method and include issue's epic_issue association + # for each issue only if the method returns true: + # + # Example: + # include_if_exportable: + # project: + # issues: + # - epic_issue + include_if_exportable: + project: + issues: + - :epic_issue diff --git a/lib/gitlab/import_export/project/relation_saver.rb b/lib/gitlab/import_export/project/relation_saver.rb index b40827e36f8dba2f856acae1ad22aebe9819a45a..8e91adac196659a1340f02cef764895a6bfe1b8e 100644 --- a/lib/gitlab/import_export/project/relation_saver.rb +++ b/lib/gitlab/import_export/project/relation_saver.rb @@ -32,7 +32,8 @@ def serializer project, reader.project_tree, json_writer, - exportable_path: 'project' + exportable_path: 'project', + current_user: nil ) end diff --git a/lib/gitlab/import_export/project/tree_saver.rb b/lib/gitlab/import_export/project/tree_saver.rb index 1b54e4b975e4d47650f6b67a392078db7cfdf515..bd34cd3ff6e13b87e8b697d655dfdc7ac9364af2 100644 --- a/lib/gitlab/import_export/project/tree_saver.rb +++ b/lib/gitlab/import_export/project/tree_saver.rb @@ -50,7 +50,8 @@ def stream_export reader.project_tree, json_writer, exportable_path: "project", - logger: @logger + logger: @logger, + current_user: @current_user ) Retriable.retriable(on: Net::OpenTimeout, on_retry: on_retry) do diff --git a/spec/lib/gitlab/import_export/config_spec.rb b/spec/lib/gitlab/import_export/config_spec.rb index fcb48678b881e84163a4367b7b622a23df65d8ac..8f848af8bd35fc91239b9852a8b741dcdf6dd785 100644 --- a/spec/lib/gitlab/import_export/config_spec.rb +++ b/spec/lib/gitlab/import_export/config_spec.rb @@ -21,10 +21,12 @@ end it 'parses default config' do + expected_keys = [:tree, :excluded_attributes, :included_attributes, :methods, :preloads, :export_reorders] + expected_keys << :include_if_exportable if ee + expect { subject }.not_to raise_error expect(subject).to be_a(Hash) - expect(subject.keys).to contain_exactly( - :tree, :excluded_attributes, :included_attributes, :methods, :preloads, :export_reorders) + expect(subject.keys).to match_array(expected_keys) end end end diff --git a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb index b8d18718dfbd0728af0f7229b798e8079aa5038b..02ac8065c9fefc446b659e398c8ac76f8e8b349c 100644 --- a/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb +++ b/spec/lib/gitlab/import_export/json/streaming_serializer_spec.rb @@ -32,18 +32,20 @@ let(:hash) { { name: exportable.name, description: exportable.description }.stringify_keys } let(:include) { [] } let(:custom_orderer) { nil } + let(:include_if_exportable) { {} } let(:relations_schema) do { only: [:name, :description], include: include, preload: { issues: nil }, - export_reorder: custom_orderer + export_reorder: custom_orderer, + include_if_exportable: include_if_exportable } end subject do - described_class.new(exportable, relations_schema, json_writer, exportable_path: exportable_path, logger: logger) + described_class.new(exportable, relations_schema, json_writer, exportable_path: exportable_path, logger: logger, current_user: user) end describe '#execute' do @@ -210,6 +212,63 @@ subject.execute end end + + describe 'conditional export of included associations' do + let(:include) do + [{ issues: { include: [{ label_links: { include: [:label] } }] } }] + end + + let(:include_if_exportable) do + { issues: [:label_links] } + end + + let_it_be(:label) { create(:label, project: exportable) } + let_it_be(:link) { create(:label_link, label: label, target: issue) } + + context 'when association is exportable' do + before do + allow_next_found_instance_of(Issue) do |issue| + allow(issue).to receive(:exportable_association?).with(:label_links, current_user: user).and_return(true) + end + end + + it 'includes exportable association' do + expected_issue = issue.to_json(include: [{ label_links: { include: [:label] } }]) + + expect(json_writer).to receive(:write_relation_array).with(exportable_path, :issues, array_including(expected_issue)) + + subject.execute + end + end + + context 'when association is not exportable' do + before do + allow_next_found_instance_of(Issue) do |issue| + allow(issue).to receive(:exportable_association?).with(:label_links, current_user: user).and_return(false) + end + end + + it 'filters out not exportable association' do + expect(json_writer).to receive(:write_relation_array).with(exportable_path, :issues, array_including(issue.to_json)) + + subject.execute + end + end + + context 'when association does not respond to exportable_association?' do + before do + allow_next_found_instance_of(Issue) do |issue| + allow(issue).to receive(:respond_to?).with(:exportable_association?).and_return(false) + end + end + + it 'filters out not exportable association' do + expect(json_writer).to receive(:write_relation_array).with(exportable_path, :issues, array_including(issue.to_json)) + + subject.execute + end + end + end end describe '#serialize_relation' do diff --git a/spec/services/bulk_imports/tree_export_service_spec.rb b/spec/services/bulk_imports/tree_export_service_spec.rb index ffb81fe2b5f6effd420ad413b4aa3860cb814182..6e26cb6dc2b3d3a7f1cb302f2b18c1009a9ec7a1 100644 --- a/spec/services/bulk_imports/tree_export_service_spec.rb +++ b/spec/services/bulk_imports/tree_export_service_spec.rb @@ -8,7 +8,7 @@ let(:relation) { 'issues' } - subject(:service) { described_class.new(project, export_path, relation) } + subject(:service) { described_class.new(project, export_path, relation, project.owner) } describe '#execute' do it 'executes export service and archives exported data' do @@ -21,7 +21,7 @@ context 'when unsupported relation is passed' do it 'raises an error' do - service = described_class.new(project, export_path, 'unsupported') + service = described_class.new(project, export_path, 'unsupported', project.owner) expect { service.execute }.to raise_error(BulkImports::Error, 'Unsupported relation export type') end