From 76349c3110a686a33f1274aba6eec79f5c320e30 Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Tue, 5 Feb 2019 19:07:32 -0200
Subject: [PATCH 01/10] Add a method to retrieve registries when selective sync
 is enabled

This method allow us to enable selective sync support for the
FDW queries to count synced registries.
---
 ee/app/models/concerns/geo/selective_sync.rb  | 17 ++++++
 ee/app/models/geo/fdw/geo_node.rb             | 61 +++++++++++++++++++
 .../models/geo/fdw/geo_node_namespace_link.rb | 12 ++++
 ee/app/models/geo/fdw/namespace.rb            | 14 +++++
 ee/app/models/geo_node.rb                     | 13 +---
 .../geo/fdw/geo_node_namespace_link_spec.rb   | 10 +++
 ee/spec/models/geo/fdw/geo_node_spec.rb       | 53 ++++++++++++++++
 ee/spec/models/geo/fdw/namespace_spec.rb      | 10 +++
 8 files changed, 178 insertions(+), 12 deletions(-)
 create mode 100644 ee/app/models/concerns/geo/selective_sync.rb
 create mode 100644 ee/app/models/geo/fdw/geo_node.rb
 create mode 100644 ee/app/models/geo/fdw/geo_node_namespace_link.rb
 create mode 100644 ee/app/models/geo/fdw/namespace.rb
 create mode 100644 ee/spec/models/geo/fdw/geo_node_namespace_link_spec.rb
 create mode 100644 ee/spec/models/geo/fdw/geo_node_spec.rb
 create mode 100644 ee/spec/models/geo/fdw/namespace_spec.rb

diff --git a/ee/app/models/concerns/geo/selective_sync.rb b/ee/app/models/concerns/geo/selective_sync.rb
new file mode 100644
index 00000000000000..3049f5d9bd0c4f
--- /dev/null
+++ b/ee/app/models/concerns/geo/selective_sync.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Geo::SelectiveSync
+  extend ActiveSupport::Concern
+
+  def selective_sync?
+    selective_sync_type.present?
+  end
+
+  def selective_sync_by_namespaces?
+    selective_sync_type == 'namespaces'
+  end
+
+  def selective_sync_by_shards?
+    selective_sync_type == 'shards'
+  end
+end
diff --git a/ee/app/models/geo/fdw/geo_node.rb b/ee/app/models/geo/fdw/geo_node.rb
new file mode 100644
index 00000000000000..93a24229cb7f3c
--- /dev/null
+++ b/ee/app/models/geo/fdw/geo_node.rb
@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+module Geo
+  module Fdw
+    class GeoNode < ::Geo::BaseFdw
+      include ::Geo::SelectiveSync
+
+      self.primary_key = :id
+      self.inheritance_column = nil
+      self.table_name = Gitlab::Geo::Fdw.foreign_table_name('geo_nodes')
+
+      serialize :selective_sync_shards, Array # rubocop:disable Cop/ActiveRecordSerialize
+
+      has_many :geo_node_namespace_links, class_name: 'Geo::Fdw::GeoNodeNamespaceLink'
+      has_many :namespaces, class_name: 'Geo::Fdw::Namespace', through: :geo_node_namespace_links
+
+      def project_registries
+        return Geo::ProjectRegistry.all unless selective_sync?
+
+        if selective_sync_by_namespaces?
+          registries_for_selected_namespaces
+        elsif selective_sync_by_shards?
+          registries_for_selected_shards
+        else
+          Geo::ProjectRegistry.none
+        end
+      end
+
+      private
+
+      def registries_for_selected_namespaces
+        query = Gitlab::ObjectHierarchy.new(namespaces).base_and_descendants
+
+        Geo::ProjectRegistry
+          .joins(fdw_inner_join_projects)
+          .where(fdw_projects_table.name => { namespace_id: query.select(:id) })
+      end
+
+      def registries_for_selected_shards
+        Geo::ProjectRegistry
+          .joins(fdw_inner_join_projects)
+          .where(fdw_projects_table.name => { repository_storage: selective_sync_shards })
+      end
+
+      def project_registries_table
+        Geo::ProjectRegistry.arel_table
+      end
+
+      def fdw_projects_table
+        Geo::Fdw::Project.arel_table
+      end
+
+      def fdw_inner_join_projects
+        project_registries_table
+          .join(fdw_projects_table, Arel::Nodes::InnerJoin)
+          .on(project_registries_table[:project_id].eq(fdw_projects_table[:id]))
+          .join_sources
+      end
+    end
+  end
+end
diff --git a/ee/app/models/geo/fdw/geo_node_namespace_link.rb b/ee/app/models/geo/fdw/geo_node_namespace_link.rb
new file mode 100644
index 00000000000000..77c9bc12194705
--- /dev/null
+++ b/ee/app/models/geo/fdw/geo_node_namespace_link.rb
@@ -0,0 +1,12 @@
+# frozen_string_literal: true
+
+module Geo
+  module Fdw
+    class GeoNodeNamespaceLink < ::Geo::BaseFdw
+      self.table_name = Gitlab::Geo::Fdw.foreign_table_name('geo_node_namespace_links')
+
+      belongs_to :geo_node, class_name: 'Geo::Fdw::GeoNode', inverse_of: :namespaces
+      belongs_to :namespace, class_name: 'Geo::Fdw::Namespace', inverse_of: :geo_nodes
+    end
+  end
+end
diff --git a/ee/app/models/geo/fdw/namespace.rb b/ee/app/models/geo/fdw/namespace.rb
new file mode 100644
index 00000000000000..729f649e8da7d4
--- /dev/null
+++ b/ee/app/models/geo/fdw/namespace.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+module Geo
+  module Fdw
+    class Namespace < ::Geo::BaseFdw
+      self.primary_key = :id
+      self.inheritance_column = nil
+      self.table_name = Gitlab::Geo::Fdw.foreign_table_name('namespaces')
+
+      has_many :geo_node_namespace_links, class_name: 'Geo::Fdw::GeoNodeNamespaceLink'
+      has_many :geo_nodes, class_name: 'Geo::Fdw::GeoNode', through: :geo_node_namespace_links
+    end
+  end
+end
diff --git a/ee/app/models/geo_node.rb b/ee/app/models/geo_node.rb
index c30734344418ae..d33d0952c9ffc0 100644
--- a/ee/app/models/geo_node.rb
+++ b/ee/app/models/geo_node.rb
@@ -2,6 +2,7 @@
 
 class GeoNode < ActiveRecord::Base
   include Presentable
+  include Geo::SelectiveSync
 
   SELECTIVE_SYNC_TYPES = %w[namespaces shards].freeze
 
@@ -213,24 +214,12 @@ def projects
     end
   end
 
-  def selective_sync_by_namespaces?
-    selective_sync_type == 'namespaces'
-  end
-
-  def selective_sync_by_shards?
-    selective_sync_type == 'shards'
-  end
-
   def projects_include?(project_id)
     return true unless selective_sync?
 
     projects.where(id: project_id).exists?
   end
 
-  def selective_sync?
-    selective_sync_type.present?
-  end
-
   def replication_slots_count
     return unless Gitlab::Database.replication_slots_supported? && primary?
 
diff --git a/ee/spec/models/geo/fdw/geo_node_namespace_link_spec.rb b/ee/spec/models/geo/fdw/geo_node_namespace_link_spec.rb
new file mode 100644
index 00000000000000..e868b2712575ff
--- /dev/null
+++ b/ee/spec/models/geo/fdw/geo_node_namespace_link_spec.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::Fdw::GeoNodeNamespaceLink, :geo, type: :model do
+  context 'relationships' do
+    it { is_expected.to belong_to(:geo_node).class_name('Geo::Fdw::GeoNode').inverse_of(:namespaces) }
+    it { is_expected.to belong_to(:namespace).class_name('Geo::Fdw::Namespace').inverse_of(:geo_nodes) }
+  end
+end
diff --git a/ee/spec/models/geo/fdw/geo_node_spec.rb b/ee/spec/models/geo/fdw/geo_node_spec.rb
new file mode 100644
index 00000000000000..0dbbbab2c6b0de
--- /dev/null
+++ b/ee/spec/models/geo/fdw/geo_node_spec.rb
@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::Fdw::GeoNode, :geo, type: :model do
+  context 'relationships' do
+    it { is_expected.to have_many(:geo_node_namespace_links).class_name('Geo::Fdw::GeoNodeNamespaceLink') }
+    it { is_expected.to have_many(:namespaces).class_name('Geo::Fdw::Namespace').through(:geo_node_namespace_links) }
+  end
+
+  # Disable transactions via :delete method because a foreign table
+  # can't see changes inside a transaction of a different connection.
+  describe '#project_registries', :delete do
+    before do
+      skip('FDW is not configured') unless Gitlab::Geo::Fdw.enabled?
+    end
+
+    let(:node) { create(:geo_node) }
+    let(:group_1) { create(:group) }
+    let(:group_2) { create(:group) }
+    let(:nested_group_1) { create(:group, parent: group_1) }
+    let(:project_1) { create(:project, group: group_1) }
+    let(:project_2) { create(:project, group: nested_group_1) }
+    let(:project_3) { create(:project, :broken_storage, group: group_2) }
+    let!(:registry_1) { create(:geo_project_registry, project: project_1) }
+    let!(:registry_2) { create(:geo_project_registry, project: project_2) }
+    let!(:registry_3) { create(:geo_project_registry, project: project_3) }
+
+    subject { described_class.find(node.id) }
+
+    it 'returns all registries without selective sync' do
+      expect(subject.project_registries).to match_array([registry_1, registry_2, registry_3])
+    end
+
+    it 'returns registries where projects belong to the namespaces with selective sync by namespace' do
+      node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
+
+      expect(subject.project_registries).to match_array([registry_1, registry_2])
+    end
+
+    it 'returns registries where projects belong to the shards with selective sync by shard' do
+      node.update!(selective_sync_type: 'shards', selective_sync_shards: %w[default bar])
+
+      expect(subject.project_registries).to match_array([registry_1, registry_2])
+    end
+
+    it 'returns nothing if an unrecognised selective sync type is used' do
+      node.update_attribute(:selective_sync_type, 'unknown')
+
+      expect(subject.project_registries).to be_empty
+    end
+  end
+end
diff --git a/ee/spec/models/geo/fdw/namespace_spec.rb b/ee/spec/models/geo/fdw/namespace_spec.rb
new file mode 100644
index 00000000000000..5f7923de4a53f2
--- /dev/null
+++ b/ee/spec/models/geo/fdw/namespace_spec.rb
@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::Fdw::Namespace, :geo, type: :model do
+  context 'relationships' do
+    it { is_expected.to have_many(:geo_node_namespace_links).class_name('Geo::Fdw::GeoNodeNamespaceLink') }
+    it { is_expected.to have_many(:geo_nodes).class_name('Geo::Fdw::GeoNode').through(:geo_node_namespace_links) }
+  end
+end
-- 
GitLab


From 900f41cb434af4d1314bf08a12f32d59c5d0c1c1 Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Tue, 5 Feb 2019 21:40:22 -0200
Subject: [PATCH 02/10] Add selective sync support for the FDW query to count
 synced registries

These changes also refactor the project registry finder to
make it easier to remove the legacy queries in the future.
---
 .../geo/legacy_project_registry_finder.rb     |  49 ++++++++
 ee/app/finders/geo/project_registry_finder.rb |  54 +++------
 .../geo/project_registry_synced_finder.rb     |  41 +++++++
 .../geo/project_registry_finder_spec.rb       | 106 +++++++++---------
 ee/spec/support/helpers/ee/geo_helpers.rb     |   5 +
 5 files changed, 165 insertions(+), 90 deletions(-)
 create mode 100644 ee/app/finders/geo/legacy_project_registry_finder.rb
 create mode 100644 ee/app/finders/geo/project_registry_synced_finder.rb

diff --git a/ee/app/finders/geo/legacy_project_registry_finder.rb b/ee/app/finders/geo/legacy_project_registry_finder.rb
new file mode 100644
index 00000000000000..ae8411dcd86069
--- /dev/null
+++ b/ee/app/finders/geo/legacy_project_registry_finder.rb
@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+module Geo
+  class LegacyProjectRegistryFinder < RegistryFinder
+    def synced_repositories
+      if selective_sync?
+        legacy_find_synced_repositories
+      else
+        find_synced_repositories
+      end
+    end
+
+    def synced_wikis
+      if use_legacy_queries?
+        legacy_find_synced_wikis
+      else
+        find_synced_wikis
+      end
+    end
+
+    private
+
+    def legacy_find_synced_repositories
+      legacy_find_project_registries(Geo::ProjectRegistry.synced_repos)
+    end
+
+    def legacy_find_synced_wikis
+      legacy_find_project_registries(Geo::ProjectRegistry.synced_wikis)
+    end
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def legacy_find_project_registries(project_registries)
+      legacy_inner_join_registry_ids(
+        current_node.projects,
+        project_registries.pluck(:project_id),
+        Project
+      )
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+
+    def find_synced_repositories
+      Geo::ProjectRegistry.synced_repos
+    end
+
+    def find_synced_wikis
+      Geo::ProjectRegistry.synced_wikis
+    end
+  end
+end
diff --git a/ee/app/finders/geo/project_registry_finder.rb b/ee/app/finders/geo/project_registry_finder.rb
index d250c834a7c540..038da8504a1298 100644
--- a/ee/app/finders/geo/project_registry_finder.rb
+++ b/ee/app/finders/geo/project_registry_finder.rb
@@ -2,16 +2,18 @@
 
 module Geo
   class ProjectRegistryFinder < RegistryFinder
+    include Gitlab::Utils::StrongMemoize
+
     def count_projects
       current_node.projects.count
     end
 
     def count_synced_repositories
       relation =
-        if selective_sync?
-          legacy_find_synced_repositories
+        if use_fdw_queries_for_selective_sync_enabled?
+          Geo::ProjectRegistrySyncedFinder.new(:repository).execute
         else
-          find_synced_repositories
+          legacy_registry_finder.synced_repositories
         end
 
       relation.count
@@ -19,10 +21,10 @@ def count_synced_repositories
 
     def count_synced_wikis
       relation =
-        if use_legacy_queries?
-          legacy_find_synced_wikis
+        if use_fdw_queries_for_selective_sync_enabled?
+          Geo::ProjectRegistrySyncedFinder.new(:wiki).execute
         else
-          fdw_find_synced_wikis
+          legacy_registry_finder.synced_wikis
         end
 
       relation.count
@@ -143,8 +145,15 @@ def find_projects_updated_recently(batch_size:)
 
     protected
 
-    def find_synced_repositories
-      Geo::ProjectRegistry.synced_repos
+    def use_fdw_queries_for_selective_sync_enabled?
+      strong_memoize(:use_fdw_queries_for_selective_sync) do
+        Gitlab::Geo::Fdw.enabled? &&
+          Feature.enabled?(:use_fdw_queries_for_selective_sync)
+      end
+    end
+
+    def legacy_registry_finder
+      Geo::LegacyProjectRegistryFinder.new(current_node: current_node)
     end
 
     def find_verified_repositories
@@ -196,11 +205,6 @@ def fdw_find_unsynced_projects
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
-    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>]
-    def fdw_find_synced_wikis
-      Geo::ProjectRegistry.synced_wikis
-    end
-
     # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
     # rubocop: disable CodeReuse/ActiveRecord
     def fdw_find_projects_updated_recently
@@ -290,37 +294,15 @@ def quote_value(value)
       ::Gitlab::SQL::Glob.q(value)
     end
 
-    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of synced projects
-    def legacy_find_synced_repositories
-      legacy_find_project_registries(Geo::ProjectRegistry.synced_repos)
-    end
-
-    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of synced projects
-    # rubocop: disable CodeReuse/ActiveRecord
-    def legacy_find_synced_wikis
-      legacy_inner_join_registry_ids(
-        current_node.projects,
-          Geo::ProjectRegistry.synced_wikis.pluck(:project_id),
-          Project
-      )
-    end
-    # rubocop: enable CodeReuse/ActiveRecord
-
     # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of verified projects
     def legacy_find_verified_repositories
       legacy_find_project_registries(Geo::ProjectRegistry.verified_repos)
     end
 
     # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of verified wikis
-    # rubocop: disable CodeReuse/ActiveRecord
     def legacy_find_verified_wikis
-      legacy_inner_join_registry_ids(
-        current_node.projects,
-          Geo::ProjectRegistry.verified_wikis.pluck(:project_id),
-          Project
-      )
+      legacy_find_project_registries(Geo::ProjectRegistry.verified_wikis)
     end
-    # rubocop: enable CodeReuse/ActiveRecord
 
     # @return [ActiveRecord::Relation<Project>] list of synced projects
     # rubocop: disable CodeReuse/ActiveRecord
diff --git a/ee/app/finders/geo/project_registry_synced_finder.rb b/ee/app/finders/geo/project_registry_synced_finder.rb
new file mode 100644
index 00000000000000..4e1d6bf94915dc
--- /dev/null
+++ b/ee/app/finders/geo/project_registry_synced_finder.rb
@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+# Finder for retrieving project registries that have been synced
+# scoped to a type (repository or wiki).
+#
+# Basic usage:
+#
+#     Geo::ProjectRegistrySyncedFinder.new(:repository).execute
+#
+# Valid `type` values are:
+#
+# * `:repository`
+# * `:wiki`
+#
+# Any other value will be ignored.
+module Geo
+  class ProjectRegistrySyncedFinder
+    attr_reader :type
+
+    def initialize(type)
+      @type = type.to_sym
+    end
+
+    def execute
+      case type
+      when :repository
+        current_node.registries.synced_repos
+      when :wiki
+        current_node.registries.synced_wikis
+      else
+        Geo::ProjectRegistry.none
+      end
+    end
+
+    private
+
+    def current_node
+      Geo::Fdw::GeoNode.current_node
+    end
+  end
+end
diff --git a/ee/spec/finders/geo/project_registry_finder_spec.rb b/ee/spec/finders/geo/project_registry_finder_spec.rb
index 1b949dd3edbb1a..3b83a965fe61dd 100644
--- a/ee/spec/finders/geo/project_registry_finder_spec.rb
+++ b/ee/spec/finders/geo/project_registry_finder_spec.rb
@@ -26,12 +26,6 @@
 
   shared_examples 'counts all the things' do
     describe '#count_synced_repositories' do
-      it 'delegates to #find_synced_repositories' do
-        expect(subject).to receive(:find_synced_repositories).and_call_original
-
-        subject.count_synced_repositories
-      end
-
       it 'counts repositories that have been synced' do
         create(:geo_project_registry, :sync_failed)
         create(:geo_project_registry, :synced, project: project_synced)
@@ -41,23 +35,10 @@
         expect(subject.count_synced_repositories).to eq 2
       end
 
-      it 'counts synced wikis with nil wiki_access_level (which means enabled wiki)' do
-        project_synced.project_feature.update!(wiki_access_level: nil)
-
-        create(:geo_project_registry, :synced, project: project_synced)
-
-        expect(subject.count_synced_wikis).to eq 1
-      end
-
       context 'with selective sync' do
         before do
           secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
-        end
-
-        it 'delegates to #legacy_find_synced_repositories' do
-          expect(subject).to receive(:legacy_find_synced_repositories).and_call_original
-
-          subject.count_synced_repositories
+          stub_fdw_current_geo_node(secondary) if Gitlab::Geo::Fdw.enabled?
         end
 
         it 'counts projects that has been synced' do
@@ -74,49 +55,51 @@
     end
 
     describe '#count_synced_wikis' do
-      it 'delegates to the correct method' do
-        expect(subject).to receive("#{method_prefix}_find_synced_wikis".to_sym).and_call_original
-
-        subject.count_synced_wikis
-      end
-
-      it 'counts wiki that have been synced' do
-        create(:geo_project_registry, :sync_failed)
-        create(:geo_project_registry, :synced, project: project_synced)
-        create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
-        create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
+      context 'with use_fdw_queries_for_selective_sync disabled' do
+        before do
+          stub_feature_flags(use_fdw_queries_for_selective_sync: false)
+        end
 
-        expect(subject.count_synced_wikis).to eq 2
-      end
+        it 'counts wiki that have been synced' do
+          create(:geo_project_registry, :sync_failed)
+          create(:geo_project_registry, :synced, project: project_synced)
+          create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
+          create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
 
-      it 'counts synced wikis with nil wiki_access_level (which means enabled wiki)' do
-        project_synced.project_feature.update!(wiki_access_level: nil)
+          expect(subject.count_synced_wikis).to eq 2
+        end
 
-        create(:geo_project_registry, :synced, project: project_synced)
+        it 'counts synced wikis with nil wiki_access_level (which means enabled wiki)' do
+          project_synced.project_feature.update!(wiki_access_level: nil)
 
-        expect(subject.count_synced_wikis).to eq 1
-      end
+          create(:geo_project_registry, :synced, project: project_synced)
 
-      context 'with selective sync' do
-        before do
-          secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
+          expect(subject.count_synced_wikis).to eq 1
         end
 
-        it 'delegates to #legacy_find_synced_wiki' do
-          expect(subject).to receive(:legacy_find_synced_wikis).and_call_original
+        context 'with selective sync' do
+          before do
+            secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
+          end
 
-          subject.count_synced_wikis
-        end
+          it 'delegates to Geo::LegacyProjectRegistryFinder#synced_wikis' do
+            expect_next_instance_of(Geo::LegacyProjectRegistryFinder) do |finder|
+              allow(finder).to receive(:synced_wikis).and_call_original
+            end
 
-        it 'counts projects that has been synced' do
-          project_1_in_synced_group = create(:project, group: synced_group)
-          project_2_in_synced_group = create(:project, group: synced_group)
+            subject.count_synced_wikis
+          end
 
-          create(:geo_project_registry, :synced, project: project_synced)
-          create(:geo_project_registry, :synced, project: project_1_in_synced_group)
-          create(:geo_project_registry, :sync_failed, project: project_2_in_synced_group)
+          it 'counts projects that has been synced' do
+            project_1_in_synced_group = create(:project, group: synced_group)
+            project_2_in_synced_group = create(:project, group: synced_group)
 
-          expect(subject.count_synced_wikis).to eq 1
+            create(:geo_project_registry, :synced, project: project_synced)
+            create(:geo_project_registry, :synced, project: project_1_in_synced_group)
+            create(:geo_project_registry, :sync_failed, project: project_2_in_synced_group)
+
+            expect(subject.count_synced_wikis).to eq 1
+          end
         end
       end
     end
@@ -639,7 +622,22 @@
       skip('FDW is not configured') if Gitlab::Database.postgresql? && !Gitlab::Geo::Fdw.enabled?
     end
 
-    include_examples 'counts all the things'
+    context 'with use_fdw_queries_for_selective_sync disabled' do
+      before do
+        stub_feature_flags(use_fdw_queries_for_selective_sync: false)
+      end
+
+      include_examples 'counts all the things'
+    end
+
+    context 'with use_fdw_queries_for_selective_sync enabled' do
+      before do
+        stub_feature_flags(use_fdw_queries_for_selective_sync: true)
+        stub_fdw_current_geo_node(secondary)
+      end
+
+      include_examples 'counts all the things'
+    end
 
     include_examples 'finds all the things' do
       let(:method_prefix) { 'fdw' }
@@ -648,7 +646,7 @@
 
   context 'Legacy' do
     before do
-      allow(Gitlab::Geo::Fdw).to receive(:enabled?).and_return(false)
+      stub_fdw_disabled
     end
 
     include_examples 'counts all the things'
diff --git a/ee/spec/support/helpers/ee/geo_helpers.rb b/ee/spec/support/helpers/ee/geo_helpers.rb
index 4cb2fdc1d9a20e..556d285131ac8a 100644
--- a/ee/spec/support/helpers/ee/geo_helpers.rb
+++ b/ee/spec/support/helpers/ee/geo_helpers.rb
@@ -17,5 +17,10 @@ def stub_secondary_node
     def stub_fdw_disabled
       allow(::Gitlab::Geo::Fdw).to receive(:enabled?).and_return(false)
     end
+
+    def stub_fdw_current_geo_node(node)
+      allow(Geo::Fdw::GeoNode).to receive(:current_node)
+        .and_return(Geo::Fdw::GeoNode.find(node.id))
+    end
   end
 end
-- 
GitLab


From 14482b190d53ad2a2f4afcd1e5c39bcacc372fca Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Wed, 6 Feb 2019 01:05:16 -0200
Subject: [PATCH 03/10] Refactor methods to count synced registries

These changes also refactor the finder to make it easier
to remove the legacy queries in the future.
---
 .../geo/legacy_project_registry_finder.rb     | 49 ---------------
 .../legacy_project_registry_synced_finder.rb  | 58 ++++++++++++++++++
 ee/app/finders/geo/project_registry_finder.rb | 32 +++++-----
 .../geo/project_registry_synced_finder.rb     | 15 ++---
 .../geo/project_registry_finder_spec.rb       | 60 +++++++------------
 ee/spec/support/helpers/ee/geo_helpers.rb     |  5 --
 6 files changed, 99 insertions(+), 120 deletions(-)
 delete mode 100644 ee/app/finders/geo/legacy_project_registry_finder.rb
 create mode 100644 ee/app/finders/geo/legacy_project_registry_synced_finder.rb

diff --git a/ee/app/finders/geo/legacy_project_registry_finder.rb b/ee/app/finders/geo/legacy_project_registry_finder.rb
deleted file mode 100644
index ae8411dcd86069..00000000000000
--- a/ee/app/finders/geo/legacy_project_registry_finder.rb
+++ /dev/null
@@ -1,49 +0,0 @@
-# frozen_string_literal: true
-
-module Geo
-  class LegacyProjectRegistryFinder < RegistryFinder
-    def synced_repositories
-      if selective_sync?
-        legacy_find_synced_repositories
-      else
-        find_synced_repositories
-      end
-    end
-
-    def synced_wikis
-      if use_legacy_queries?
-        legacy_find_synced_wikis
-      else
-        find_synced_wikis
-      end
-    end
-
-    private
-
-    def legacy_find_synced_repositories
-      legacy_find_project_registries(Geo::ProjectRegistry.synced_repos)
-    end
-
-    def legacy_find_synced_wikis
-      legacy_find_project_registries(Geo::ProjectRegistry.synced_wikis)
-    end
-
-    # rubocop: disable CodeReuse/ActiveRecord
-    def legacy_find_project_registries(project_registries)
-      legacy_inner_join_registry_ids(
-        current_node.projects,
-        project_registries.pluck(:project_id),
-        Project
-      )
-    end
-    # rubocop: enable CodeReuse/ActiveRecord
-
-    def find_synced_repositories
-      Geo::ProjectRegistry.synced_repos
-    end
-
-    def find_synced_wikis
-      Geo::ProjectRegistry.synced_wikis
-    end
-  end
-end
diff --git a/ee/app/finders/geo/legacy_project_registry_synced_finder.rb b/ee/app/finders/geo/legacy_project_registry_synced_finder.rb
new file mode 100644
index 00000000000000..1e65e220996913
--- /dev/null
+++ b/ee/app/finders/geo/legacy_project_registry_synced_finder.rb
@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+# Finder for retrieving project registries that have been synced
+# scoped to a type (repository or wiki) using cross-database joins
+# for selective sync.
+#
+# Basic usage:
+#
+#     Geo::LegacyProjectRegistrySyncedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
+#
+# Valid `type` values are:
+#
+# * `:repository`
+# * `:wiki`
+#
+# Any other value will be ignored.
+module Geo
+  class LegacyProjectRegistrySyncedFinder < RegistryFinder
+    def initialize(current_node:, type:)
+      super(current_node: current_node)
+      @type = type.to_sym
+    end
+
+    def execute
+      if selective_sync?
+        legacy_registries_for_synced_projects
+      else
+        registries_for_synced_projects
+      end
+    end
+
+    private
+
+    attr_reader :type
+
+    def registries_for_synced_projects
+      case type
+      when :repository
+        Geo::ProjectRegistry.synced_repos
+      when :wiki
+        Geo::ProjectRegistry.synced_wikis
+      else
+        Geo::ProjectRegistry.none
+      end
+    end
+
+    # rubocop: disable CodeReuse/ActiveRecord
+    def legacy_registries_for_synced_projects
+      legacy_inner_join_registry_ids(
+        registries_for_synced_projects,
+        current_node.projects.pluck(:id),
+        Geo::ProjectRegistry,
+        foreign_key: :project_id
+      )
+    end
+    # rubocop: enable CodeReuse/ActiveRecord
+  end
+end
diff --git a/ee/app/finders/geo/project_registry_finder.rb b/ee/app/finders/geo/project_registry_finder.rb
index 038da8504a1298..ba6ceb2a405275 100644
--- a/ee/app/finders/geo/project_registry_finder.rb
+++ b/ee/app/finders/geo/project_registry_finder.rb
@@ -9,25 +9,11 @@ def count_projects
     end
 
     def count_synced_repositories
-      relation =
-        if use_fdw_queries_for_selective_sync_enabled?
-          Geo::ProjectRegistrySyncedFinder.new(:repository).execute
-        else
-          legacy_registry_finder.synced_repositories
-        end
-
-      relation.count
+      registries_for_synced_projects(:repository).count
     end
 
     def count_synced_wikis
-      relation =
-        if use_fdw_queries_for_selective_sync_enabled?
-          Geo::ProjectRegistrySyncedFinder.new(:wiki).execute
-        else
-          legacy_registry_finder.synced_wikis
-        end
-
-      relation.count
+      registries_for_synced_projects(:wiki).count
     end
 
     def count_failed_repositories
@@ -152,8 +138,18 @@ def use_fdw_queries_for_selective_sync_enabled?
       end
     end
 
-    def legacy_registry_finder
-      Geo::LegacyProjectRegistryFinder.new(current_node: current_node)
+    def finder_klass_for_synced_registries
+      if use_fdw_queries_for_selective_sync_enabled?
+        Geo::ProjectRegistrySyncedFinder
+      else
+        Geo::LegacyProjectRegistrySyncedFinder
+      end
+    end
+
+    def registries_for_synced_projects(type)
+      finder_klass_for_synced_registries
+        .new(current_node: current_node, type: type)
+        .execute
     end
 
     def find_verified_repositories
diff --git a/ee/app/finders/geo/project_registry_synced_finder.rb b/ee/app/finders/geo/project_registry_synced_finder.rb
index 4e1d6bf94915dc..d0ecb7bd3c1711 100644
--- a/ee/app/finders/geo/project_registry_synced_finder.rb
+++ b/ee/app/finders/geo/project_registry_synced_finder.rb
@@ -1,11 +1,11 @@
 # frozen_string_literal: true
 
 # Finder for retrieving project registries that have been synced
-# scoped to a type (repository or wiki).
+# scoped to a type (repository or wiki) using FDW queries.
 #
 # Basic usage:
 #
-#     Geo::ProjectRegistrySyncedFinder.new(:repository).execute
+#     Geo::ProjectRegistrySyncedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
 #
 # Valid `type` values are:
 #
@@ -15,9 +15,10 @@
 # Any other value will be ignored.
 module Geo
   class ProjectRegistrySyncedFinder
-    attr_reader :type
+    attr_reader :current_node, :type
 
-    def initialize(type)
+    def initialize(current_node:, type:)
+      @current_node = Geo::Fdw::GeoNode.find(current_node.id)
       @type = type.to_sym
     end
 
@@ -31,11 +32,5 @@ def execute
         Geo::ProjectRegistry.none
       end
     end
-
-    private
-
-    def current_node
-      Geo::Fdw::GeoNode.current_node
-    end
   end
 end
diff --git a/ee/spec/finders/geo/project_registry_finder_spec.rb b/ee/spec/finders/geo/project_registry_finder_spec.rb
index 3b83a965fe61dd..b0f621d735e1ec 100644
--- a/ee/spec/finders/geo/project_registry_finder_spec.rb
+++ b/ee/spec/finders/geo/project_registry_finder_spec.rb
@@ -38,7 +38,6 @@
       context 'with selective sync' do
         before do
           secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
-          stub_fdw_current_geo_node(secondary) if Gitlab::Geo::Fdw.enabled?
         end
 
         it 'counts projects that has been synced' do
@@ -55,51 +54,37 @@
     end
 
     describe '#count_synced_wikis' do
-      context 'with use_fdw_queries_for_selective_sync disabled' do
-        before do
-          stub_feature_flags(use_fdw_queries_for_selective_sync: false)
-        end
+      it 'counts wiki that have been synced' do
+        create(:geo_project_registry, :sync_failed)
+        create(:geo_project_registry, :synced, project: project_synced)
+        create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
+        create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
 
-        it 'counts wiki that have been synced' do
-          create(:geo_project_registry, :sync_failed)
-          create(:geo_project_registry, :synced, project: project_synced)
-          create(:geo_project_registry, :synced, :repository_dirty, project: project_repository_dirty)
-          create(:geo_project_registry, :synced, :wiki_dirty, project: project_wiki_dirty)
+        expect(subject.count_synced_wikis).to eq 2
+      end
 
-          expect(subject.count_synced_wikis).to eq 2
-        end
+      it 'counts synced wikis with nil wiki_access_level (which means enabled wiki)' do
+        project_synced.project_feature.update!(wiki_access_level: nil)
 
-        it 'counts synced wikis with nil wiki_access_level (which means enabled wiki)' do
-          project_synced.project_feature.update!(wiki_access_level: nil)
+        create(:geo_project_registry, :synced, project: project_synced)
 
-          create(:geo_project_registry, :synced, project: project_synced)
+        expect(subject.count_synced_wikis).to eq 1
+      end
 
-          expect(subject.count_synced_wikis).to eq 1
+      context 'with selective sync' do
+        before do
+          secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
         end
 
-        context 'with selective sync' do
-          before do
-            secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
-          end
-
-          it 'delegates to Geo::LegacyProjectRegistryFinder#synced_wikis' do
-            expect_next_instance_of(Geo::LegacyProjectRegistryFinder) do |finder|
-              allow(finder).to receive(:synced_wikis).and_call_original
-            end
-
-            subject.count_synced_wikis
-          end
-
-          it 'counts projects that has been synced' do
-            project_1_in_synced_group = create(:project, group: synced_group)
-            project_2_in_synced_group = create(:project, group: synced_group)
+        it 'counts projects that has been synced' do
+          project_1_in_synced_group = create(:project, group: synced_group)
+          project_2_in_synced_group = create(:project, group: synced_group)
 
-            create(:geo_project_registry, :synced, project: project_synced)
-            create(:geo_project_registry, :synced, project: project_1_in_synced_group)
-            create(:geo_project_registry, :sync_failed, project: project_2_in_synced_group)
+          create(:geo_project_registry, :synced, project: project_synced)
+          create(:geo_project_registry, :synced, project: project_1_in_synced_group)
+          create(:geo_project_registry, :sync_failed, project: project_2_in_synced_group)
 
-            expect(subject.count_synced_wikis).to eq 1
-          end
+          expect(subject.count_synced_wikis).to eq 1
         end
       end
     end
@@ -633,7 +618,6 @@
     context 'with use_fdw_queries_for_selective_sync enabled' do
       before do
         stub_feature_flags(use_fdw_queries_for_selective_sync: true)
-        stub_fdw_current_geo_node(secondary)
       end
 
       include_examples 'counts all the things'
diff --git a/ee/spec/support/helpers/ee/geo_helpers.rb b/ee/spec/support/helpers/ee/geo_helpers.rb
index 556d285131ac8a..4cb2fdc1d9a20e 100644
--- a/ee/spec/support/helpers/ee/geo_helpers.rb
+++ b/ee/spec/support/helpers/ee/geo_helpers.rb
@@ -17,10 +17,5 @@ def stub_secondary_node
     def stub_fdw_disabled
       allow(::Gitlab::Geo::Fdw).to receive(:enabled?).and_return(false)
     end
-
-    def stub_fdw_current_geo_node(node)
-      allow(Geo::Fdw::GeoNode).to receive(:current_node)
-        .and_return(Geo::Fdw::GeoNode.find(node.id))
-    end
   end
 end
-- 
GitLab


From 70198db70aadb62a229fce2127bdab19f7b4fdc9 Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Wed, 6 Feb 2019 01:06:15 -0200
Subject: [PATCH 04/10] Add spec for Geo::ProjectRegistrySyncedFinder

---
 .../project_registry_synced_finder_spec.rb    | 89 +++++++++++++++++++
 1 file changed, 89 insertions(+)
 create mode 100644 ee/spec/finders/geo/project_registry_synced_finder_spec.rb

diff --git a/ee/spec/finders/geo/project_registry_synced_finder_spec.rb b/ee/spec/finders/geo/project_registry_synced_finder_spec.rb
new file mode 100644
index 00000000000000..58de40fe7719f7
--- /dev/null
+++ b/ee/spec/finders/geo/project_registry_synced_finder_spec.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Geo::ProjectRegistrySyncedFinder, :geo do
+  # Disable transactions via :delete method because a foreign table
+  # can't see changes inside a transaction of a different connection.
+  describe '#execute', :delete do
+    let(:node) { create(:geo_node) }
+    let(:group_1) { create(:group) }
+    let(:group_2) { create(:group) }
+    let(:nested_group_1) { create(:group, parent: group_1) }
+    let(:project_1) { create(:project, group: group_1) }
+    let(:project_2) { create(:project, group: nested_group_1) }
+    let(:project_3) { create(:project, group: nested_group_1) }
+    let(:project_4) { create(:project, :broken_storage, group: group_2) }
+    let(:project_5) { create(:project, :broken_storage, group: group_2) }
+    let!(:registry_synced) { create(:geo_project_registry, :synced, project: project_1) }
+    let!(:registry_repository_dirty) { create(:geo_project_registry, :synced, :repository_dirty, project: project_2) }
+    let!(:registry_wiki_dirty) { create(:geo_project_registry, :synced, :wiki_dirty, project: project_3) }
+    let!(:registry_wiki_dirty_broken_shard) { create(:geo_project_registry, :synced, :wiki_dirty, project: project_4) }
+    let!(:registry_repository_dirty_broken_shard) { create(:geo_project_registry, :synced, :repository_dirty, project: project_5) }
+    let!(:registry_sync_failed) { create(:geo_project_registry, :sync_failed) }
+
+    before do
+      skip('FDW is not configured') unless Gitlab::Geo::Fdw.enabled?
+    end
+
+    context 'with repository type' do
+      subject { described_class.new(current_node: node, type: :repository) }
+
+      context 'without selective sync' do
+        it 'returns all synced registries' do
+          expect(subject.execute).to match_array([registry_synced, registry_wiki_dirty, registry_wiki_dirty_broken_shard])
+        end
+      end
+
+      context 'with selective sync by namespace' do
+        it 'returns synced registries where projects belongs to the namespaces' do
+          node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
+
+          expect(subject.execute).to match_array([registry_synced, registry_wiki_dirty])
+        end
+      end
+
+      context 'with selective sync by shard' do
+        it 'returns synced registries where projects belongs to the shards' do
+          node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
+
+          expect(subject.execute).to match_array([registry_wiki_dirty_broken_shard])
+        end
+      end
+    end
+
+    context 'with wiki type' do
+      subject { described_class.new(current_node: node, type: :wiki) }
+
+      context 'without selective sync' do
+        it 'returns all synced registries' do
+          expect(subject.execute).to match_array([registry_synced, registry_repository_dirty, registry_repository_dirty_broken_shard])
+        end
+      end
+
+      context 'with selective sync by namespace' do
+        it 'returns synced registries where projects belongs to the namespaces' do
+          node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
+
+          expect(subject.execute).to match_array([registry_synced, registry_repository_dirty])
+        end
+      end
+
+      context 'with selective sync by shard' do
+        it 'returns synced registries where projects belongs to the shards' do
+          node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
+
+          expect(subject.execute).to match_array([registry_repository_dirty_broken_shard])
+        end
+      end
+    end
+
+    context 'with invalid type' do
+      subject { described_class.new(current_node: node, type: :invalid) }
+
+      it 'returns nothing' do
+        expect(subject.execute).to be_empty
+      end
+    end
+  end
+end
-- 
GitLab


From 86938886cc7711effbf37ea5621112ef41788a8e Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Wed, 6 Feb 2019 01:07:10 -0200
Subject: [PATCH 05/10] Add spec for Geo::LegacyProjectRegistrySyncedFinder

---
 ...acy_project_registry_synced_finder_spec.rb | 105 ++++++++++++++++++
 1 file changed, 105 insertions(+)
 create mode 100644 ee/spec/finders/geo/legacy_project_registry_synced_finder_spec.rb

diff --git a/ee/spec/finders/geo/legacy_project_registry_synced_finder_spec.rb b/ee/spec/finders/geo/legacy_project_registry_synced_finder_spec.rb
new file mode 100644
index 00000000000000..881a6637336ccf
--- /dev/null
+++ b/ee/spec/finders/geo/legacy_project_registry_synced_finder_spec.rb
@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Geo::LegacyProjectRegistrySyncedFinder, :geo do
+  include EE::GeoHelpers
+
+  describe '#execute' do
+    let(:node) { create(:geo_node) }
+    let(:group_1) { create(:group) }
+    let(:group_2) { create(:group) }
+    let(:nested_group_1) { create(:group, parent: group_1) }
+    let(:project_1) { create(:project, group: group_1) }
+    let(:project_2) { create(:project, group: nested_group_1) }
+    let(:project_3) { create(:project, group: nested_group_1) }
+    let(:project_4) { create(:project, :broken_storage, group: group_2) }
+    let(:project_5) { create(:project, :broken_storage, group: group_2) }
+    let!(:registry_synced) { create(:geo_project_registry, :synced, project: project_1) }
+    let!(:registry_repository_dirty) { create(:geo_project_registry, :synced, :repository_dirty, project: project_2) }
+    let!(:registry_wiki_dirty) { create(:geo_project_registry, :synced, :wiki_dirty, project: project_3) }
+    let!(:registry_wiki_dirty_broken_shard) { create(:geo_project_registry, :synced, :wiki_dirty, project: project_4) }
+    let!(:registry_repository_dirty_broken_shard) { create(:geo_project_registry, :synced, :repository_dirty, project: project_5) }
+    let!(:registry_sync_failed) { create(:geo_project_registry, :sync_failed) }
+
+    shared_examples 'finds synced registries' do
+      context 'with repository type' do
+        subject { described_class.new(current_node: node, type: :repository) }
+
+        context 'without selective sync' do
+          it 'returns all synced registries' do
+            expect(subject.execute).to match_array([registry_synced, registry_wiki_dirty, registry_wiki_dirty_broken_shard])
+          end
+        end
+
+        context 'with selective sync by namespace' do
+          it 'returns synced registries where projects belongs to the namespaces' do
+            node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
+
+            expect(subject.execute).to match_array([registry_synced, registry_wiki_dirty])
+          end
+        end
+
+        context 'with selective sync by shard' do
+          it 'returns synced registries where projects belongs to the shards' do
+            node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
+
+            expect(subject.execute).to match_array([registry_wiki_dirty_broken_shard])
+          end
+        end
+      end
+
+      context 'with wiki type' do
+        subject { described_class.new(current_node: node, type: :wiki) }
+
+        context 'without selective sync' do
+          it 'returns all synced registries' do
+            expect(subject.execute).to match_array([registry_synced, registry_repository_dirty, registry_repository_dirty_broken_shard])
+          end
+        end
+
+        context 'with selective sync by namespace' do
+          it 'returns synced registries where projects belongs to the namespaces' do
+            node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
+
+            expect(subject.execute).to match_array([registry_synced, registry_repository_dirty])
+          end
+        end
+
+        context 'with selective sync by shard' do
+          it 'returns synced registries where projects belongs to the shards' do
+            node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
+
+            expect(subject.execute).to match_array([registry_repository_dirty_broken_shard])
+          end
+        end
+      end
+
+      context 'with invalid type' do
+        subject { described_class.new(current_node: node, type: :invalid) }
+
+        it 'returns nothing' do
+          expect(subject.execute).to be_empty
+        end
+      end
+    end
+
+    # Disable transactions via :delete method because a foreign table
+    # can't see changes inside a transaction of a different connection.
+    context 'FDW', :delete do
+      before do
+        skip('FDW is not configured') unless Gitlab::Geo::Fdw.enabled?
+      end
+
+      include_examples 'finds synced registries'
+    end
+
+    context 'Legacy' do
+      before do
+        stub_fdw_disabled
+      end
+
+      include_examples 'finds synced registries'
+    end
+  end
+end
-- 
GitLab


From 2999aeabd3a02093debb59cc066d5502559e3f4b Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Wed, 6 Feb 2019 23:42:49 -0200
Subject: [PATCH 06/10] Add scope to find synced registries to
 Geo::ProjectRegistry

---
 .../legacy_project_registry_synced_finder.rb  | 19 ++++++-------------
 .../geo/project_registry_synced_finder.rb     | 15 +++++----------
 ee/app/models/geo/project_registry.rb         | 11 +++++++++++
 3 files changed, 22 insertions(+), 23 deletions(-)

diff --git a/ee/app/finders/geo/legacy_project_registry_synced_finder.rb b/ee/app/finders/geo/legacy_project_registry_synced_finder.rb
index 1e65e220996913..80eff69c44a17a 100644
--- a/ee/app/finders/geo/legacy_project_registry_synced_finder.rb
+++ b/ee/app/finders/geo/legacy_project_registry_synced_finder.rb
@@ -23,9 +23,9 @@ def initialize(current_node:, type:)
 
     def execute
       if selective_sync?
-        legacy_registries_for_synced_projects
+        synced_registries_for_selective_sync
       else
-        registries_for_synced_projects
+        synced_registries
       end
     end
 
@@ -33,21 +33,14 @@ def execute
 
     attr_reader :type
 
-    def registries_for_synced_projects
-      case type
-      when :repository
-        Geo::ProjectRegistry.synced_repos
-      when :wiki
-        Geo::ProjectRegistry.synced_wikis
-      else
-        Geo::ProjectRegistry.none
-      end
+    def synced_registries
+      Geo::ProjectRegistry.synced(type)
     end
 
     # rubocop: disable CodeReuse/ActiveRecord
-    def legacy_registries_for_synced_projects
+    def synced_registries_for_selective_sync
       legacy_inner_join_registry_ids(
-        registries_for_synced_projects,
+        synced_registries,
         current_node.projects.pluck(:id),
         Geo::ProjectRegistry,
         foreign_key: :project_id
diff --git a/ee/app/finders/geo/project_registry_synced_finder.rb b/ee/app/finders/geo/project_registry_synced_finder.rb
index d0ecb7bd3c1711..0b655ff157a43f 100644
--- a/ee/app/finders/geo/project_registry_synced_finder.rb
+++ b/ee/app/finders/geo/project_registry_synced_finder.rb
@@ -15,22 +15,17 @@
 # Any other value will be ignored.
 module Geo
   class ProjectRegistrySyncedFinder
-    attr_reader :current_node, :type
-
     def initialize(current_node:, type:)
       @current_node = Geo::Fdw::GeoNode.find(current_node.id)
       @type = type.to_sym
     end
 
     def execute
-      case type
-      when :repository
-        current_node.registries.synced_repos
-      when :wiki
-        current_node.registries.synced_wikis
-      else
-        Geo::ProjectRegistry.none
-      end
+      current_node.project_registries.synced(type)
     end
+
+    private
+
+    attr_reader :current_node, :type
   end
 end
diff --git a/ee/app/models/geo/project_registry.rb b/ee/app/models/geo/project_registry.rb
index 385915a4c88aa9..0885003741406e 100644
--- a/ee/app/models/geo/project_registry.rb
+++ b/ee/app/models/geo/project_registry.rb
@@ -88,6 +88,17 @@ def self.with_search(query)
     where(project: Geo::Fdw::Project.search(query))
   end
 
+  def self.synced(type)
+    case type
+    when :repository
+      synced_repos
+    when :wiki
+      synced_wikis
+    else
+      none
+    end
+  end
+
   def self.flag_repositories_for_resync!
     update_all(
       resync_repository: true,
-- 
GitLab


From c3e3fe39e29e3a21a7ef9e2551be5a8c5a83b7b9 Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Mon, 11 Feb 2019 16:20:57 -0200
Subject: [PATCH 07/10] Add method to check if FDW queries is enabled for
 selective sync

---
 ee/app/finders/geo/project_registry_finder.rb | 11 +-----
 ee/lib/gitlab/geo/fdw.rb                      |  4 +++
 ee/spec/lib/gitlab/geo/fdw_spec.rb            | 34 +++++++++++++++++++
 3 files changed, 39 insertions(+), 10 deletions(-)

diff --git a/ee/app/finders/geo/project_registry_finder.rb b/ee/app/finders/geo/project_registry_finder.rb
index ba6ceb2a405275..542cf193dce1dd 100644
--- a/ee/app/finders/geo/project_registry_finder.rb
+++ b/ee/app/finders/geo/project_registry_finder.rb
@@ -2,8 +2,6 @@
 
 module Geo
   class ProjectRegistryFinder < RegistryFinder
-    include Gitlab::Utils::StrongMemoize
-
     def count_projects
       current_node.projects.count
     end
@@ -131,15 +129,8 @@ def find_projects_updated_recently(batch_size:)
 
     protected
 
-    def use_fdw_queries_for_selective_sync_enabled?
-      strong_memoize(:use_fdw_queries_for_selective_sync) do
-        Gitlab::Geo::Fdw.enabled? &&
-          Feature.enabled?(:use_fdw_queries_for_selective_sync)
-      end
-    end
-
     def finder_klass_for_synced_registries
-      if use_fdw_queries_for_selective_sync_enabled?
+      if Gitlab::Geo::Fdw.enabled_for_selective_sync?
         Geo::ProjectRegistrySyncedFinder
       else
         Geo::LegacyProjectRegistrySyncedFinder
diff --git a/ee/lib/gitlab/geo/fdw.rb b/ee/lib/gitlab/geo/fdw.rb
index 96e8702b7d969b..0013c478546193 100644
--- a/ee/lib/gitlab/geo/fdw.rb
+++ b/ee/lib/gitlab/geo/fdw.rb
@@ -19,6 +19,10 @@ def enabled?
           value.nil? ? true : value
         end
 
+        def enabled_for_selective_sync?
+          enabled? && Feature.enabled?(:use_fdw_queries_for_selective_sync)
+        end
+
         # Return full table name with foreign schema
         #
         # @param [String] table_name
diff --git a/ee/spec/lib/gitlab/geo/fdw_spec.rb b/ee/spec/lib/gitlab/geo/fdw_spec.rb
index 7600ecdc6a1671..7e3823d160e382 100644
--- a/ee/spec/lib/gitlab/geo/fdw_spec.rb
+++ b/ee/spec/lib/gitlab/geo/fdw_spec.rb
@@ -44,6 +44,40 @@
     end
   end
 
+  describe '.enabled_for_selective_sync?' do
+    context 'when the feature flag is enabled' do
+      before do
+        stub_feature_flags(use_fdw_queries_for_selective_sync: true)
+      end
+
+      it 'returns false when FDW is disabled' do
+        allow(described_class).to receive(:enabled?).and_return(false)
+
+        expect(described_class.enabled_for_selective_sync?).to eq false
+      end
+
+      it 'returns true when FDW is enabled' do
+        expect(described_class.enabled_for_selective_sync?).to eq true
+      end
+    end
+
+    context 'when the feature flag is disabled' do
+      before do
+        stub_feature_flags(use_fdw_queries_for_selective_sync: false)
+      end
+
+      it 'returns false when FDW is disabled' do
+        allow(described_class).to receive(:enabled?).and_return(false)
+
+        expect(described_class.enabled_for_selective_sync?).to eq false
+      end
+
+      it 'returns false when FDW is enabled' do
+        expect(described_class.enabled_for_selective_sync?).to eq false
+      end
+    end
+  end
+
   describe '.foreign_tables_up_to_date?' do
     it 'returns false when foreign schema does not exist' do
       drop_foreign_schema
-- 
GitLab


From 41721e46628322c532e590a59587e506f02aa1ce Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Mon, 11 Feb 2019 17:02:01 -0200
Subject: [PATCH 08/10] Disable transactions via :delete method

Disable transactions via :delete method because a foreign table
can't see changes inside a transaction of a different connection.
---
 ee/spec/models/geo_node_status_spec.rb | 4 +++-
 ee/spec/tasks/geo_rake_spec.rb         | 4 +++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/ee/spec/models/geo_node_status_spec.rb b/ee/spec/models/geo_node_status_spec.rb
index a10b4415427d35..519bf14d15765a 100644
--- a/ee/spec/models/geo_node_status_spec.rb
+++ b/ee/spec/models/geo_node_status_spec.rb
@@ -1,6 +1,8 @@
 require 'spec_helper'
 
-describe GeoNodeStatus, :geo do
+# Disable transactions via :delete method because a foreign table
+# can't see changes inside a transaction of a different connection.
+describe GeoNodeStatus, :geo, :delete do
   include ::EE::GeoHelpers
 
   let!(:primary) { create(:geo_node, :primary) }
diff --git a/ee/spec/tasks/geo_rake_spec.rb b/ee/spec/tasks/geo_rake_spec.rb
index e505225ee639d1..ff5035a56b7b74 100644
--- a/ee/spec/tasks/geo_rake_spec.rb
+++ b/ee/spec/tasks/geo_rake_spec.rb
@@ -58,7 +58,9 @@
     end
   end
 
-  describe 'status task' do
+  # Disable transactions via :delete method because a foreign table
+  # can't see changes inside a transaction of a different connection.
+  describe 'status task', :delete do
     let!(:current_node) { create(:geo_node) }
     let!(:primary_node) { create(:geo_node, :primary) }
     let!(:geo_event_log) { create(:geo_event_log) }
-- 
GitLab


From 4b1ad09f7f497a4799f896c710b1b9e2c1342694 Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Mon, 11 Feb 2019 17:16:39 -0200
Subject: [PATCH 09/10] Add CHANGELOG entry

---
 ...nt-selective-sync-support-for-the-various-fdw-queries.yml | 5 +++++
 1 file changed, 5 insertions(+)
 create mode 100644 ee/changelogs/unreleased/8798-geo-implement-selective-sync-support-for-the-various-fdw-queries.yml

diff --git a/ee/changelogs/unreleased/8798-geo-implement-selective-sync-support-for-the-various-fdw-queries.yml b/ee/changelogs/unreleased/8798-geo-implement-selective-sync-support-for-the-various-fdw-queries.yml
new file mode 100644
index 00000000000000..6f0efa2487cd78
--- /dev/null
+++ b/ee/changelogs/unreleased/8798-geo-implement-selective-sync-support-for-the-various-fdw-queries.yml
@@ -0,0 +1,5 @@
+---
+title: Geo - Add selective sync support for the FDW queries to count synced registries
+merge_request: 9445
+author:
+type: changed
-- 
GitLab


From b2feec7573e35d7a2264bc19593f3d6ba5cebce1 Mon Sep 17 00:00:00 2001
From: Douglas Barbosa Alexandre <dbalexandre@gmail.com>
Date: Thu, 28 Feb 2019 20:06:15 -0300
Subject: [PATCH 10/10] Use a custom recursive CTE query to retrieve selected
 namespaces

---
 ee/app/models/geo/fdw/geo_node.rb | 33 ++++++++++++++++++++++++++++++-
 1 file changed, 32 insertions(+), 1 deletion(-)

diff --git a/ee/app/models/geo/fdw/geo_node.rb b/ee/app/models/geo/fdw/geo_node.rb
index 93a24229cb7f3c..87c4f333fe5d70 100644
--- a/ee/app/models/geo/fdw/geo_node.rb
+++ b/ee/app/models/geo/fdw/geo_node.rb
@@ -29,13 +29,36 @@ def project_registries
       private
 
       def registries_for_selected_namespaces
-        query = Gitlab::ObjectHierarchy.new(namespaces).base_and_descendants
+        query = selected_namespaces_and_descendants
 
         Geo::ProjectRegistry
           .joins(fdw_inner_join_projects)
           .where(fdw_projects_table.name => { namespace_id: query.select(:id) })
       end
 
+      def selected_namespaces_and_descendants
+        relation = selected_namespaces_and_descendants_cte.apply_to(Geo::Fdw::Namespace.all)
+        relation.extend(Gitlab::Database::ReadOnlyRelation)
+        relation
+      end
+
+      def selected_namespaces_and_descendants_cte
+        cte = Gitlab::SQL::RecursiveCTE.new(:base_and_descendants)
+
+        cte << geo_node_namespace_links
+          .select(fdw_geo_node_namespace_links_table[:namespace_id].as('id'))
+          .except(:order)
+
+        # Recursively get all the descendants of the base set.
+        cte << Geo::Fdw::Namespace
+          .select(fdw_namespaces_table[:id])
+          .from([fdw_namespaces_table, cte.table])
+          .where(fdw_namespaces_table[:parent_id].eq(cte.table[:id]))
+          .except(:order)
+
+        cte
+      end
+
       def registries_for_selected_shards
         Geo::ProjectRegistry
           .joins(fdw_inner_join_projects)
@@ -50,6 +73,14 @@ def fdw_projects_table
         Geo::Fdw::Project.arel_table
       end
 
+      def fdw_namespaces_table
+        Geo::Fdw::Namespace.arel_table
+      end
+
+      def fdw_geo_node_namespace_links_table
+        Geo::Fdw::GeoNodeNamespaceLink.arel_table
+      end
+
       def fdw_inner_join_projects
         project_registries_table
           .join(fdw_projects_table, Arel::Nodes::InnerJoin)
-- 
GitLab