diff --git a/.rubocop_manual_todo.yml b/.rubocop_manual_todo.yml
index 61d08cffddc29225c32b11b4149e234d46987c52..131b56f81322873b143a8f3a258149bfdcb54543 100644
--- a/.rubocop_manual_todo.yml
+++ b/.rubocop_manual_todo.yml
@@ -25,7 +25,6 @@ FactoryBot/InlineAssociation:
     - 'spec/factories/go_modules.rb'
     - 'spec/factories/group_group_links.rb'
     - 'spec/factories/import_export_uploads.rb'
-    - 'spec/factories/notes.rb'
     - 'spec/factories/uploads.rb'
     - 'spec/factories/wiki_pages.rb'
 
diff --git a/spec/factories/notes.rb b/spec/factories/notes.rb
index 4b1f3194ce5e37f6a810f8f50b25f8c6607183ce..299d08972b76f5af345d0015cd79f34a379aa9d0 100644
--- a/spec/factories/notes.rb
+++ b/spec/factories/notes.rb
@@ -8,7 +8,7 @@
   factory :note do
     project
     note { generate(:title) }
-    author { project&.creator || create(:user) }
+    author { project&.creator || association(:user) }
     on_issue
 
     factory :note_on_commit,             traits: [:on_commit]
@@ -55,7 +55,7 @@
       end
 
       position do
-        build(:text_diff_position,
+        association(:text_diff_position,
               file: "files/ruby/popen.rb",
               old_line: nil,
               new_line: line_number,
@@ -64,7 +64,7 @@
 
       trait :folded_position do
         position do
-          build(:text_diff_position,
+          association(:text_diff_position,
                 file: "files/ruby/popen.rb",
                 old_line: 1,
                 new_line: 1,
@@ -74,7 +74,7 @@
 
       factory :image_diff_note_on_merge_request do
         position do
-          build(:image_diff_position,
+          association(:image_diff_position,
                 file: "files/images/any_image.png",
                 diff_refs: diff_refs)
         end
@@ -90,7 +90,7 @@
       end
 
       position do
-        build(:text_diff_position,
+        association(:text_diff_position,
           file: "files/ruby/popen.rb",
           old_line: nil,
           new_line: line_number,
@@ -100,7 +100,11 @@
     end
 
     factory :diff_note_on_design, parent: :note, traits: [:on_design], class: 'DiffNote' do
-      position { build(:image_diff_position, file: noteable.full_path, diff_refs: noteable.diff_refs) }
+      position do
+        association(:image_diff_position,
+              file: noteable.full_path,
+              diff_refs: noteable.diff_refs)
+      end
     end
 
     trait :on_commit do
diff --git a/spec/models/note_spec.rb b/spec/models/note_spec.rb
index 6e87ca6dcf760ed2ac988359bff3235610d1ea89..5c14a3db54effd9201a9839f71962c346b5b13ce 100644
--- a/spec/models/note_spec.rb
+++ b/spec/models/note_spec.rb
@@ -890,35 +890,31 @@ def retrieve_participants
   describe '#cache_markdown_field' do
     let(:html) { '<p>some html</p>'}
 
+    before do
+      allow(Banzai::Renderer).to receive(:cacheless_render_field).and_call_original
+    end
+
     context 'note for a project snippet' do
       let(:snippet) { create(:project_snippet) }
-      let(:note) { build(:note_on_project_snippet, project: snippet.project, noteable: snippet) }
+      let(:note) { create(:note_on_project_snippet, project: snippet.project, noteable: snippet) }
 
-      before do
+      it 'skips project check' do
         expect(Banzai::Renderer).to receive(:cacheless_render_field)
-          .with(note, :note, { skip_project_check: false }).and_return(html)
+          .with(note, :note, { skip_project_check: false })
 
-        note.save
-      end
-
-      it 'creates a note' do
-        expect(note.note_html).to eq(html)
+        note.update!(note: html)
       end
     end
 
     context 'note for a personal snippet' do
       let(:snippet) { create(:personal_snippet) }
-      let(:note) { build(:note_on_personal_snippet, noteable: snippet) }
+      let(:note) { create(:note_on_personal_snippet, noteable: snippet) }
 
-      before do
+      it 'does not skip project check' do
         expect(Banzai::Renderer).to receive(:cacheless_render_field)
-          .with(note, :note, { skip_project_check: true }).and_return(html)
-
-        note.save
-      end
+          .with(note, :note, { skip_project_check: true })
 
-      it 'creates a note' do
-        expect(note.note_html).to eq(html)
+        note.update!(note: html)
       end
     end
   end
diff --git a/spec/support/factory_bot.rb b/spec/support/factory_bot.rb
index c9d372993b57eae0f66923cc3203445b28de61e6..5761e05d5410eda8d97fc09de6c2817d53fd6000 100644
--- a/spec/support/factory_bot.rb
+++ b/spec/support/factory_bot.rb
@@ -3,3 +3,16 @@
 FactoryBot::SyntaxRunner.class_eval do
   include RSpec::Mocks::ExampleMethods
 end
+
+# Patching FactoryBot to allow stubbing non AR models
+# See https://github.com/thoughtbot/factory_bot/pull/1466
+module Gitlab
+  module FactoryBotStubPatch
+    def has_settable_id?(result_instance)
+      result_instance.class.respond_to?(:primary_key) &&
+        result_instance.class.primary_key
+    end
+  end
+end
+
+FactoryBot::Strategy::Stub.prepend(Gitlab::FactoryBotStubPatch)