From 94bc72847ab20c47ad611735201b85fbce6f402b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Wed, 2 Oct 2019 17:13:42 +0200
Subject: [PATCH 01/10] Refactor Project Restore

Improves where we store test JSONs.
---
 .../project.json}                             |  0
 .../project_tree_restorer_spec.rb             |  3 +-
 .../import_export/{ => complex}/project.json  |  0
 .../project.json}                             |  0
 .../project.json}                             |  0
 .../project.json}                             |  0
 .../project_tree_restorer_spec.rb             | 91 ++++++++++---------
 .../relation_rename_service_spec.rb           | 10 +-
 spec/support/import_export/common_util.rb     |  7 ++
 9 files changed, 61 insertions(+), 50 deletions(-)
 rename ee/spec/fixtures/lib/gitlab/import_export/{project.designs.json => designs/project.json} (100%)
 rename spec/fixtures/lib/gitlab/import_export/{ => complex}/project.json (100%)
 rename spec/fixtures/lib/gitlab/import_export/{project.group.json => group/project.json} (100%)
 rename spec/fixtures/lib/gitlab/import_export/{project.light.json => light/project.json} (100%)
 rename spec/fixtures/lib/gitlab/import_export/{project.milestone-iid.json => milestone-iid/project.json} (100%)

diff --git a/ee/spec/fixtures/lib/gitlab/import_export/project.designs.json b/ee/spec/fixtures/lib/gitlab/import_export/designs/project.json
similarity index 100%
rename from ee/spec/fixtures/lib/gitlab/import_export/project.designs.json
rename to ee/spec/fixtures/lib/gitlab/import_export/designs/project.json
diff --git a/ee/spec/lib/ee/gitlab/import_export/project_tree_restorer_spec.rb b/ee/spec/lib/ee/gitlab/import_export/project_tree_restorer_spec.rb
index c912f9643ceb02..9abcc677d2a288 100644
--- a/ee/spec/lib/ee/gitlab/import_export/project_tree_restorer_spec.rb
+++ b/ee/spec/lib/ee/gitlab/import_export/project_tree_restorer_spec.rb
@@ -12,8 +12,7 @@
   let(:restored_project_json) { project_tree_restorer.restore }
 
   before do
-    allow(shared).to receive(:export_path).and_return('spec/fixtures/lib/gitlab/import_export/')
-    project_tree_restorer.instance_variable_set(:@path, 'ee/spec/fixtures/lib/gitlab/import_export/project.designs.json')
+    setup_import_export_config('designs', 'ee')
   end
 
   describe 'restoring design management data' do
diff --git a/spec/fixtures/lib/gitlab/import_export/project.json b/spec/fixtures/lib/gitlab/import_export/complex/project.json
similarity index 100%
rename from spec/fixtures/lib/gitlab/import_export/project.json
rename to spec/fixtures/lib/gitlab/import_export/complex/project.json
diff --git a/spec/fixtures/lib/gitlab/import_export/project.group.json b/spec/fixtures/lib/gitlab/import_export/group/project.json
similarity index 100%
rename from spec/fixtures/lib/gitlab/import_export/project.group.json
rename to spec/fixtures/lib/gitlab/import_export/group/project.json
diff --git a/spec/fixtures/lib/gitlab/import_export/project.light.json b/spec/fixtures/lib/gitlab/import_export/light/project.json
similarity index 100%
rename from spec/fixtures/lib/gitlab/import_export/project.light.json
rename to spec/fixtures/lib/gitlab/import_export/light/project.json
diff --git a/spec/fixtures/lib/gitlab/import_export/project.milestone-iid.json b/spec/fixtures/lib/gitlab/import_export/milestone-iid/project.json
similarity index 100%
rename from spec/fixtures/lib/gitlab/import_export/project.milestone-iid.json
rename to spec/fixtures/lib/gitlab/import_export/milestone-iid/project.json
diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb
index 88a7fbea1d12b6..073cf45d6883f6 100644
--- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb
@@ -2,6 +2,8 @@
 include ImportExport::CommonUtil
 
 describe Gitlab::ImportExport::ProjectTreeRestorer do
+  include ImportExport::CommonUtil
+
   let(:shared) { project.import_export_shared }
 
   describe 'restore project tree' do
@@ -16,7 +18,8 @@
       RSpec::Mocks.with_temporary_scope do
         @project = create(:project, :builds_enabled, :issues_disabled, name: 'project', path: 'project')
         @shared = @project.import_export_shared
-        allow(@shared).to receive(:export_path).and_return('spec/fixtures/lib/gitlab/import_export/')
+
+        setup_import_export_config('complex')
 
         allow_any_instance_of(Repository).to receive(:fetch_source_branch!).and_return(true)
         allow_any_instance_of(Gitlab::Git::Repository).to receive(:branch_exists?).and_return(false)
@@ -314,21 +317,33 @@
     end
   end
 
-  context 'Light JSON' do
+  context 'project.json file access check' do
     let(:user) { create(:user) }
     let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') }
     let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
     let(:restored_project_json) { project_tree_restorer.restore }
 
-    before do
-      allow(shared).to receive(:export_path).and_return('spec/fixtures/lib/gitlab/import_export/')
+    it 'does not read a symlink' do
+      Dir.mktmpdir do |tmpdir|
+        setup_symlink(tmpdir, 'project.json')
+        allow(shared).to receive(:export_path).and_call_original
+
+        expect(project_tree_restorer.restore).to eq(false)
+        expect(shared.errors).to include('Incorrect JSON format')
+      end
     end
+  end
+
+  context 'Light JSON' do
+    let(:user) { create(:user) }
+    let!(:project) { create(:project, :builds_disabled, :issues_disabled, name: 'project', path: 'project') }
+    let(:project_tree_restorer) { described_class.new(user: user, shared: shared, project: project) }
+    let(:restored_project_json) { project_tree_restorer.restore }
 
     context 'with a simple project' do
       before do
-        project_tree_restorer.instance_variable_set(:@path, "spec/fixtures/lib/gitlab/import_export/project.light.json")
-
-        restored_project_json
+        setup_import_export_config('light')
+        expect(restored_project_json).to eq(true)
       end
 
       it_behaves_like 'restores project correctly',
@@ -339,19 +354,6 @@
                       first_issue_labels: 1,
                       services: 1
 
-      context 'project.json file access check' do
-        it 'does not read a symlink' do
-          Dir.mktmpdir do |tmpdir|
-            setup_symlink(tmpdir, 'project.json')
-            allow(shared).to receive(:export_path).and_call_original
-
-            restored_project_json
-
-            expect(shared.errors).to be_empty
-          end
-        end
-      end
-
       context 'when there is an existing build with build token' do
         before do
           create(:ci_build, token: 'abcd')
@@ -367,6 +369,10 @@
     end
 
     context 'when the project has overridden params in import data' do
+      before do
+        setup_import_export_config('light')
+      end
+
       it 'handles string versions of visibility_level' do
         # Project needs to be in a group for visibility level comparison
         # to happen
@@ -375,24 +381,21 @@
 
         project.create_import_data(data: { override_params: { visibility_level: Gitlab::VisibilityLevel::INTERNAL.to_s } })
 
-        restored_project_json
-
+        expect(restored_project_json).to eq(true)
         expect(project.visibility_level).to eq(Gitlab::VisibilityLevel::INTERNAL)
       end
 
       it 'overwrites the params stored in the JSON' do
         project.create_import_data(data: { override_params: { description: "Overridden" } })
 
-        restored_project_json
-
+        expect(restored_project_json).to eq(true)
         expect(project.description).to eq("Overridden")
       end
 
       it 'does not allow setting params that are excluded from import_export settings' do
         project.create_import_data(data: { override_params: { lfs_enabled: true } })
 
-        restored_project_json
-
+        expect(restored_project_json).to eq(true)
         expect(project.lfs_enabled).to be_falsey
       end
 
@@ -408,7 +411,7 @@
 
         project.create_import_data(data: { override_params: disabled_access_levels })
 
-        restored_project_json
+        expect(restored_project_json).to eq(true)
 
         aggregate_failures do
           access_level_keys.each do |key|
@@ -429,9 +432,8 @@
       end
 
       before do
-        project_tree_restorer.instance_variable_set(:@path, "spec/fixtures/lib/gitlab/import_export/project.group.json")
-
-        restored_project_json
+        setup_import_export_config('group')
+        expect(restored_project_json).to eq(true)
       end
 
       it_behaves_like 'restores project correctly',
@@ -463,11 +465,11 @@
       end
 
       before do
-        project_tree_restorer.instance_variable_set(:@path, "spec/fixtures/lib/gitlab/import_export/project.light.json")
+        setup_import_export_config('light')
       end
 
       it 'does not import any templated services' do
-        restored_project_json
+        expect(restored_project_json).to eq(true)
 
         expect(project.services.where(template: true).count).to eq(0)
       end
@@ -477,8 +479,7 @@
 
         expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error)
 
-        restored_project_json
-
+        expect(restored_project_json).to eq(true)
         expect(project.labels.count).to eq(1)
       end
 
@@ -487,8 +488,7 @@
 
         expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error)
 
-        restored_project_json
-
+        expect(restored_project_json).to eq(true)
         expect(project.group.milestones.count).to eq(1)
         expect(project.milestones.count).to eq(0)
       end
@@ -504,13 +504,14 @@
                group: create(:group))
       end
 
-      it 'preserves the project milestone IID' do
-        project_tree_restorer.instance_variable_set(:@path, "spec/fixtures/lib/gitlab/import_export/project.milestone-iid.json")
+      before do
+        setup_import_export_config('milestone-iid')
+      end
 
+      it 'preserves the project milestone IID' do
         expect_any_instance_of(Gitlab::ImportExport::Shared).not_to receive(:error)
 
-        restored_project_json
-
+        expect(restored_project_json).to eq(true)
         expect(project.milestones.count).to eq(2)
         expect(Milestone.find_by_title('Another milestone').iid).to eq(1)
         expect(Milestone.find_by_title('Group-level milestone').iid).to eq(2)
@@ -518,19 +519,21 @@
     end
 
     context 'with external authorization classification labels' do
+      before do
+        setup_import_export_config('light')
+      end
+
       it 'converts empty external classification authorization labels to nil' do
         project.create_import_data(data: { override_params: { external_authorization_classification_label: "" } })
 
-        restored_project_json
-
+        expect(restored_project_json).to eq(true)
         expect(project.external_authorization_classification_label).to be_nil
       end
 
       it 'preserves valid external classification authorization labels' do
         project.create_import_data(data: { override_params: { external_authorization_classification_label: "foobar" } })
 
-        restored_project_json
-
+        expect(restored_project_json).to eq(true)
         expect(project.external_authorization_classification_label).to eq("foobar")
       end
     end
diff --git a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb
index 472bf55d37ed58..843de27df1a2de 100644
--- a/spec/lib/gitlab/import_export/relation_rename_service_spec.rb
+++ b/spec/lib/gitlab/import_export/relation_rename_service_spec.rb
@@ -3,6 +3,8 @@
 require 'spec_helper'
 
 describe Gitlab::ImportExport::RelationRenameService do
+  include ImportExport::CommonUtil
+
   let(:renames) do
     {
       'example_relation1' => 'new_example_relation1',
@@ -21,12 +23,12 @@
 
   context 'when importing' do
     let(:project_tree_restorer) { Gitlab::ImportExport::ProjectTreeRestorer.new(user: user, shared: shared, project: project) }
-    let(:import_path) { 'spec/fixtures/lib/gitlab/import_export' }
-    let(:file_content) { IO.read("#{import_path}/project.json") }
-    let!(:json_file) { ActiveSupport::JSON.decode(file_content) }
+    let(:file_content) { IO.read(File.join(shared.export_path, 'project.json')) }
+    let(:json_file) { ActiveSupport::JSON.decode(file_content) }
 
     before do
-      allow(shared).to receive(:export_path).and_return(import_path)
+      setup_import_export_config('complex')
+
       allow(ActiveSupport::JSON).to receive(:decode).and_call_original
       allow(ActiveSupport::JSON).to receive(:decode).with(file_content).and_return(json_file)
     end
diff --git a/spec/support/import_export/common_util.rb b/spec/support/import_export/common_util.rb
index ac6840dbcfcf4c..4e149c9fa5464f 100644
--- a/spec/support/import_export/common_util.rb
+++ b/spec/support/import_export/common_util.rb
@@ -8,5 +8,12 @@ def setup_symlink(tmpdir, symlink_name)
       File.open("#{tmpdir}/test", 'w') { |file| file.write("test") }
       FileUtils.ln_s("#{tmpdir}/test", "#{tmpdir}/#{symlink_name}")
     end
+
+    def setup_import_export_config(name, prefix = nil)
+      export_path = [prefix, 'spec', 'fixtures', 'lib', 'gitlab', 'import_export', name].compact
+      export_path = File.join(*export_path)
+
+      allow_any_instance_of(Gitlab::ImportExport).to receive(:export_path) { export_path }
+    end
   end
 end
-- 
GitLab


From 6ca0ce413f0117dee9fb5cc36983135b63c5cf94 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Kamil=20Trzci=C5=84ski?= <ayufan@ayufan.eu>
Date: Wed, 2 Oct 2019 17:13:42 +0200
Subject: [PATCH 02/10] Refactor ProjectTreeRestorer logic

This rewrites a majority of the logic
for tree restorer to make it much cleaner
in terms of relations between objects
and make it much easier to persist objects
at very explicit points.
---
 app/models/project.rb                         |  21 --
 .../import_export/project_tree_restorer.rb    | 198 ++++++++----------
 2 files changed, 92 insertions(+), 127 deletions(-)

diff --git a/app/models/project.rb b/app/models/project.rb
index b5046bbb6eecc9..535d843b15e94a 100644
--- a/app/models/project.rb
+++ b/app/models/project.rb
@@ -1962,27 +1962,6 @@ def auto_devops_variables
     (auto_devops || build_auto_devops)&.predefined_variables
   end
 
-  def append_or_update_attribute(name, value)
-    if Project.reflect_on_association(name).try(:macro) == :has_many
-      # if this is 1-to-N relation, update the parent object
-      value.each do |item|
-        item.update!(
-          Project.reflect_on_association(name).foreign_key => id)
-      end
-
-      # force to drop relation cache
-      public_send(name).reset # rubocop:disable GitlabSecurity/PublicSend
-
-      # succeeded
-      true
-    else
-      # if this is another relation or attribute, update just object
-      update_attribute(name, value)
-    end
-  rescue ActiveRecord::RecordInvalid => e
-    raise e, "Failed to set #{name}: #{e.message}"
-  end
-
   # Tries to set repository as read_only, checking for existing Git transfers in progress beforehand
   #
   # @return [Boolean] true when set to read_only or false when an existing git transfer is in progress
diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 9433a231b4a1dd..7e5a86a5601694 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -15,7 +15,6 @@ def initialize(user:, shared:, project:)
         @user = user
         @shared = shared
         @project = project
-        @saved = true
       end
 
       def restore
@@ -33,7 +32,8 @@ def restore
         ActiveRecord::Base.uncached do
           ActiveRecord::Base.no_touching do
             update_project_params!
-            create_relations
+            create_project_relations!
+            post_import!
           end
         end
 
@@ -69,77 +69,75 @@ def merge_requests_mapping
       # in the DB. The structure and relationships between models are guessed from
       # the configuration yaml file too.
       # Finally, it updates each attribute in the newly imported project.
-      def create_relations
-        project_relations.each do |relation_key, relation_definition|
-          relation_key_s = relation_key.to_s
-
-          if relation_definition.present?
-            create_sub_relations(relation_key_s, relation_definition, @tree_hash)
-          elsif @tree_hash[relation_key_s].present?
-            save_relation_hash(relation_key_s, @tree_hash[relation_key_s])
-          end
-        end
+      def create_project_relations!
+        project_relations.each(&method(
+          :process_project_relation!))
+      end
 
+      def post_import!
         @project.merge_requests.set_latest_merge_request_diff_ids!
-
-        @saved
       end
 
-      def save_relation_hash(relation_key, relation_hash_batch)
-        relation_hash = create_relation(relation_key, relation_hash_batch)
+      def process_project_relation!(relation_key, relation_definition)
+        data_hashes = @tree_hash.delete(relation_key)
+        return unless data_hashes
 
-        remove_group_models(relation_hash) if relation_hash.is_a?(Array)
+        # we do not care if we process array or hash
+        data_hashes = [data_hashes] unless data_hashes.is_a?(Array)
+
+        # consume and remove objects from memory
+        while data_hash = data_hashes.shift
+          process_project_relation_item!(relation_key, relation_definition, data_hash)
+        end
+      end
 
-        @saved = false unless @project.append_or_update_attribute(relation_key, relation_hash)
+      def process_project_relation_item!(relation_key, relation_definition, data_hash)
+        relation_object = build_relation(relation_key, relation_definition, data_hash)
+        return unless relation_object
+        return if is_group_model?(relation_object)
 
-        save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+        relation_object.project = @project
+        relation_object.save!
 
-        @project.reset
+        save_id_mapping(relation_key, data_hash, relation_object)
       end
 
       # Older, serialized CI pipeline exports may only have a
       # merge_request_id and not the full hash of the merge request. To
       # import these pipelines, we need to preserve the mapping between
       # the old and new the merge request ID.
-      def save_id_mappings(relation_key, relation_hash_batch, relation_hash)
+      def save_id_mapping(relation_key, data_hash, relation_object)
         return unless relation_key == 'merge_requests'
 
-        relation_hash = Array(relation_hash)
-
-        Array(relation_hash_batch).each_with_index do |raw_data, index|
-          merge_requests_mapping[raw_data['id']] = relation_hash[index]['id']
-        end
-      end
-
-      # Remove project models that became group models as we found them at group level.
-      # This no longer required saving them at the root project level.
-      # For example, in the case of an existing group label that matched the title.
-      def remove_group_models(relation_hash)
-        relation_hash.reject! do |value|
-          GROUP_MODELS.include?(value.class) && value.group_id
-        end
+        merge_requests_mapping[data_hash['id']] = relation_object.id
       end
 
       def project_relations
-        @project_relations ||= reader.attributes_finder.find_relations_tree(:project)
+        @project_relations ||=
+          reader
+            .attributes_finder
+            .find_relations_tree(:project)
+            .deep_stringify_keys
       end
 
       def update_project_params!
-        Gitlab::Timeless.timeless(@project) do
-          project_params = @tree_hash.reject do |key, value|
-            project_relations.include?(key.to_sym)
-          end
+        project_params = @tree_hash.reject do |key, value|
+          project_relations.include?(key)
+        end
+
+        project_params = project_params.merge(
+          present_project_override_params)
 
-          project_params = project_params.merge(present_project_override_params)
+        # Cleaning all imported and overridden params
+        project_params = Gitlab::ImportExport::AttributeCleaner.clean(
+          relation_hash: project_params,
+          relation_class: Project,
+          excluded_keys: excluded_keys_for_relation(:project))
 
-          # Cleaning all imported and overridden params
-          project_params = Gitlab::ImportExport::AttributeCleaner.clean(
-            relation_hash: project_params,
-            relation_class: Project,
-            excluded_keys: excluded_keys_for_relation(:project))
+        @project.assign_attributes(project_params)
+        @project.drop_visibility_level!
 
-          @project.assign_attributes(project_params)
-          @project.drop_visibility_level!
+        Gitlab::Timeless.timeless(@project) do
           @project.save!
         end
       end
@@ -156,73 +154,61 @@ def project_override_params
         @project_override_params ||= @project.import_data&.data&.fetch('override_params', nil) || {}
       end
 
-      # Given a relation hash containing one or more models and its relationships,
-      # loops through each model and each object from a model type and
-      # and assigns its correspondent attributes hash from +tree_hash+
-      # Example:
-      # +relation_key+ issues, loops through the list of *issues* and for each individual
-      # issue, finds any subrelations such as notes, creates them and assign them back to the hash
-      #
-      # Recursively calls this method if the sub-relation is a hash containing more sub-relations
-      def create_sub_relations(relation_key, relation_definition, tree_hash, save: true)
-        return if tree_hash[relation_key].blank?
-
-        tree_array = [tree_hash[relation_key]].flatten
-
-        # Avoid keeping a possible heavy object in memory once we are done with it
-        while relation_item = tree_array.shift
-          # The transaction at this level is less speedy than one single transaction
-          # But we can't have it in the upper level or GC won't get rid of the AR objects
-          # after we save the batch.
-          Project.transaction do
-            process_sub_relation(relation_key, relation_definition, relation_item)
-
-            # For every subrelation that hangs from Project, save the associated records altogether
-            # This effectively batches all records per subrelation item, only keeping those in memory
-            # We have to keep in mind that more batch granularity << Memory, but >> Slowness
-            if save
-              save_relation_hash(relation_key, [relation_item])
-              tree_hash[relation_key].delete(relation_item)
-            end
-          end
-        end
-
-        tree_hash.delete(relation_key) if save
+      def build_relations(relation_key, relation_definition, data_hashes)
+        data_hashes.map do |data_hash|
+          build_relation(relation_key, relation_definition, data_hash)
+        end.compact
       end
 
-      def process_sub_relation(relation_key, relation_definition, relation_item)
-        relation_definition.each do |sub_relation_key, sub_relation_definition|
-          # We just use author to get the user ID, do not attempt to create an instance.
-          next if sub_relation_key == :author
+      def build_relation(relation_key, relation_definition, data_hash)
+        # TODO: This is hack to not create relation for the author
+        # Rather make `RelationFactory#set_note_author` to take care of that
+        return data_hash if relation_key == 'author'
 
-          sub_relation_key_s = sub_relation_key.to_s
+        # create relation objects recursively for all sub-objects
+        relation_definition.each do |sub_relation_key, sub_relation_definition|
+          transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
+        end
 
-          # create dependent relations if present
-          if sub_relation_definition.present?
-            create_sub_relations(sub_relation_key_s, sub_relation_definition, relation_item, save: false)
+        Gitlab::ImportExport::RelationFactory.create(
+          relation_sym: relation_key.to_sym,
+          relation_hash: data_hash,
+          members_mapper: members_mapper,
+          merge_requests_mapping: merge_requests_mapping,
+          user: @user,
+          project: @project,
+          excluded_keys: excluded_keys_for_relation(relation_key))
+      end
+
+      def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definition)
+        sub_data_hash = data_hash[sub_relation_key]
+        return unless sub_data_hash
+
+        # if object is a hash we can create simple object
+        # as it means that this is 1-to-1 vs 1-to-many
+        sub_data_hash =
+          if sub_data_hash.is_a?(Array)
+            build_relations(
+              sub_relation_key,
+              sub_relation_definition,
+              sub_data_hash).presence
+          else
+            build_relation(
+              sub_relation_key,
+              sub_relation_definition,
+              sub_data_hash)
           end
 
-          # transform relation hash to actual object
-          sub_relation_hash = relation_item[sub_relation_key_s]
-          if sub_relation_hash.present?
-            relation_item[sub_relation_key_s] = create_relation(sub_relation_key, sub_relation_hash)
-          end
+        # persist object(s) or delete from relation
+        if sub_data_hash
+          data_hash[sub_relation_key] = sub_data_hash
+        else
+          data_hash.delete(sub_relation_key)
         end
       end
 
-      def create_relation(relation_key, relation_hash_list)
-        relation_array = [relation_hash_list].flatten.map do |relation_hash|
-          Gitlab::ImportExport::RelationFactory.create(
-            relation_sym: relation_key.to_sym,
-            relation_hash: relation_hash,
-            members_mapper: members_mapper,
-            merge_requests_mapping: merge_requests_mapping,
-            user: @user,
-            project: @project,
-            excluded_keys: excluded_keys_for_relation(relation_key))
-        end.compact
-
-        relation_hash_list.is_a?(Array) ? relation_array : relation_array.first
+      def is_group_model?(relation_object)
+        GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
       end
 
       def reader
-- 
GitLab


From 0ec072eb0c1f86fd2a3988b3c97f4739b153941e Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Thu, 10 Oct 2019 14:48:20 +0300
Subject: [PATCH 03/10] Remove redundant spec

---
 spec/models/project_spec.rb | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/spec/models/project_spec.rb b/spec/models/project_spec.rb
index 5922a6f36f56b6..908dc40683d3b8 100644
--- a/spec/models/project_spec.rb
+++ b/spec/models/project_spec.rb
@@ -3342,22 +3342,6 @@ def enable_lfs
     end
   end
 
-  describe '#append_or_update_attribute' do
-    let(:project) { create(:project) }
-
-    it 'shows full error updating an invalid MR' do
-      expect { project.append_or_update_attribute(:merge_requests, [create(:merge_request)]) }
-        .to raise_error(ActiveRecord::RecordInvalid, /Failed to set merge_requests:/)
-    end
-
-    it 'updates the project successfully' do
-      merge_request = create(:merge_request, target_project: project, source_project: project)
-
-      expect { project.append_or_update_attribute(:merge_requests, [merge_request]) }
-        .not_to raise_error
-    end
-  end
-
   describe '#update' do
     let(:project) { create(:project) }
 
-- 
GitLab


From f39506c7d61a0d0c2d4a44908598bdc6bd66cfe9 Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Fri, 11 Oct 2019 13:41:20 +0300
Subject: [PATCH 04/10] Fix rubocop offence

---
 lib/gitlab/import_export/project_tree_restorer.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 7e5a86a5601694..c401f96b5c16f0 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -94,7 +94,7 @@ def process_project_relation!(relation_key, relation_definition)
       def process_project_relation_item!(relation_key, relation_definition, data_hash)
         relation_object = build_relation(relation_key, relation_definition, data_hash)
         return unless relation_object
-        return if is_group_model?(relation_object)
+        return if group_model?(relation_object)
 
         relation_object.project = @project
         relation_object.save!
@@ -207,7 +207,7 @@ def transform_sub_relations!(data_hash, sub_relation_key, sub_relation_definitio
         end
       end
 
-      def is_group_model?(relation_object)
+      def group_model?(relation_object)
         GROUP_MODELS.include?(relation_object.class) && relation_object.group_id
       end
 
-- 
GitLab


From 422cec41059a6f7bb55faea88008abebbdcf40c2 Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Fri, 11 Oct 2019 16:46:54 +0300
Subject: [PATCH 05/10] Avoid creating ProjectCiCdSetting duplicate

---
 lib/gitlab/import_export/relation_factory.rb | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index ae8025c52efa07..ce1b4bc7edbf7d 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -38,7 +38,7 @@ class RelationFactory
 
       IMPORTED_OBJECT_MAX_RETRIES = 5.freeze
 
-      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request].freeze
+      EXISTING_OBJECT_CHECK = %i[milestone milestones label labels project_label project_labels group_label group_labels project_feature merge_request ProjectCiCdSetting].freeze
 
       TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
 
-- 
GitLab


From 64b1af851d0e3cb0ee0dcca1a56d75e29bb1e249 Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Fri, 11 Oct 2019 17:40:03 +0300
Subject: [PATCH 06/10] Call remove_feature_dependent_sub_relations! (EE)

---
 lib/gitlab/import_export/project_tree_restorer.rb | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index c401f96b5c16f0..afe770bdf9b26a 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -87,6 +87,8 @@ def process_project_relation!(relation_key, relation_definition)
 
         # consume and remove objects from memory
         while data_hash = data_hashes.shift
+          remove_feature_dependent_sub_relations!(data_hash)
+
           process_project_relation_item!(relation_key, relation_definition, data_hash)
         end
       end
-- 
GitLab


From 72c3030fc100d4de2ef74180b1326886d3ff1dac Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Fri, 11 Oct 2019 19:02:02 +0300
Subject: [PATCH 07/10] Introduce `UNIQUE_RELATIONS` in RelationFactory

---
 lib/gitlab/import_export/relation_factory.rb | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index ce1b4bc7edbf7d..cf3749573417d8 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -42,6 +42,9 @@ class RelationFactory
 
       TOKEN_RESET_MODELS = %i[Project Namespace Ci::Trigger Ci::Build Ci::Runner ProjectHook].freeze
 
+      # This represents all relations that have unique key on `project_id`
+      UNIQUE_RELATIONS = %i[project_feature ProjectCiCdSetting].freeze
+
       def self.create(*args)
         new(*args).create
       end
@@ -324,7 +327,8 @@ def legacy_trigger?
       end
 
       def find_or_create_object!
-        return relation_class.find_or_create_by(project_id: @project.id) if @relation_name == :project_feature
+        return relation_class.find_or_create_by(project_id: @project.id) if UNIQUE_RELATIONS.include?(@relation_name)
+
         return find_or_create_merge_request! if @relation_name == :merge_request
 
         # Can't use IDs as validation exists calling `group` or `project` attributes
-- 
GitLab


From 70d741660b28df74c82baecb95d22dc2f10d2ad3 Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Fri, 25 Oct 2019 14:58:34 +0300
Subject: [PATCH 08/10] Apply review suggestion

---
 lib/gitlab/import_export/project_tree_restorer.rb | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index afe770bdf9b26a..95e0abff45d758 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -87,13 +87,13 @@ def process_project_relation!(relation_key, relation_definition)
 
         # consume and remove objects from memory
         while data_hash = data_hashes.shift
-          remove_feature_dependent_sub_relations!(data_hash)
-
           process_project_relation_item!(relation_key, relation_definition, data_hash)
         end
       end
 
       def process_project_relation_item!(relation_key, relation_definition, data_hash)
+        remove_feature_dependent_sub_relations!(data_hash)
+
         relation_object = build_relation(relation_key, relation_definition, data_hash)
         return unless relation_object
         return if group_model?(relation_object)
-- 
GitLab


From 48e4ff9594f1d736834f2fb09ba9e4991463b808 Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Mon, 28 Oct 2019 12:48:54 +0300
Subject: [PATCH 09/10] Fix merge conflict resolve

`remove_feature_dependent_sub_relations!` was removed entirely in
https://gitlab.com/gitlab-org/gitlab/merge_requests/18024
---
 lib/gitlab/import_export/project_tree_restorer.rb | 2 --
 1 file changed, 2 deletions(-)

diff --git a/lib/gitlab/import_export/project_tree_restorer.rb b/lib/gitlab/import_export/project_tree_restorer.rb
index 95e0abff45d758..c401f96b5c16f0 100644
--- a/lib/gitlab/import_export/project_tree_restorer.rb
+++ b/lib/gitlab/import_export/project_tree_restorer.rb
@@ -92,8 +92,6 @@ def process_project_relation!(relation_key, relation_definition)
       end
 
       def process_project_relation_item!(relation_key, relation_definition, data_hash)
-        remove_feature_dependent_sub_relations!(data_hash)
-
         relation_object = build_relation(relation_key, relation_definition, data_hash)
         return unless relation_object
         return if group_model?(relation_object)
-- 
GitLab


From d08fbf5e64ea58998cf3f2d3fdf5810ee2bccf05 Mon Sep 17 00:00:00 2001
From: Aleksei Lipniagov <alipniagov@gitlab.com>
Date: Wed, 30 Oct 2019 17:06:01 +0300
Subject: [PATCH 10/10] Additional spec and fix for edge case

I am still investigating if this case is something considered normal in
our data model.
But in out gitlabhq tarball we have such cases.
I am also checking why this was not failing on master.
Probably we are not passing keys with empty arrays into RelationFactory
at all?
---
 lib/gitlab/import_export/relation_factory.rb  |  2 +-
 .../gitlab/import_export/complex/project.json | 19 +++++++++++++++++++
 .../project_tree_restorer_spec.rb             |  6 +++---
 3 files changed, 23 insertions(+), 4 deletions(-)

diff --git a/lib/gitlab/import_export/relation_factory.rb b/lib/gitlab/import_export/relation_factory.rb
index cf3749573417d8..f8f6c3e6107623 100644
--- a/lib/gitlab/import_export/relation_factory.rb
+++ b/lib/gitlab/import_export/relation_factory.rb
@@ -277,7 +277,7 @@ def setup_diff
       end
 
       def setup_pipeline
-        @relation_hash.fetch('stages').each do |stage|
+        @relation_hash.fetch('stages', []).each do |stage|
           stage.statuses.each do |status|
             status.pipeline = imported_object
           end
diff --git a/spec/fixtures/lib/gitlab/import_export/complex/project.json b/spec/fixtures/lib/gitlab/import_export/complex/project.json
index 864933ca1b44c0..86931d66472973 100644
--- a/spec/fixtures/lib/gitlab/import_export/complex/project.json
+++ b/spec/fixtures/lib/gitlab/import_export/complex/project.json
@@ -6680,6 +6680,25 @@
           ]
         }
       ]
+    },
+    {
+      "id": 41,
+      "project_id": 5,
+      "ref": "master",
+      "sha": "2ea1f3dec713d940208fb5ce4a38765ecb5d3f73",
+      "before_sha": null,
+      "push_data": null,
+      "created_at": "2016-03-22T15:20:35.763Z",
+      "updated_at": "2016-03-22T15:20:35.763Z",
+      "tag": null,
+      "yaml_errors": null,
+      "committed_at": null,
+      "status": "failed",
+      "started_at": null,
+      "finished_at": null,
+      "duration": null,
+      "stages": [
+      ]
     }
   ],
   "triggers": [
diff --git a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb
index 073cf45d6883f6..1269fac7f48119 100644
--- a/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb
+++ b/spec/lib/gitlab/import_export/project_tree_restorer_spec.rb
@@ -260,9 +260,9 @@
         end
 
         it 'has the correct number of pipelines and statuses' do
-          expect(@project.ci_pipelines.size).to eq(5)
+          expect(@project.ci_pipelines.size).to eq(6)
 
-          @project.ci_pipelines.zip([2, 2, 2, 2, 2])
+          @project.ci_pipelines.zip([0, 2, 2, 2, 2, 2])
             .each do |(pipeline, expected_status_size)|
             expect(pipeline.statuses.size).to eq(expected_status_size)
           end
@@ -271,7 +271,7 @@
 
       context 'when restoring hierarchy of pipeline, stages and jobs' do
         it 'restores pipelines' do
-          expect(Ci::Pipeline.all.count).to be 5
+          expect(Ci::Pipeline.all.count).to be 6
         end
 
         it 'restores pipeline stages' do
-- 
GitLab