diff --git a/ee/app/models/ee/container_repository.rb b/ee/app/models/ee/container_repository.rb index 8131475eac7c6c3ea664b6ee9a8cb7694e74d5a4..7f088c651a0e0afe1e42c59683d265af092abdac 100644 --- a/ee/app/models/ee/container_repository.rb +++ b/ee/app/models/ee/container_repository.rb @@ -23,6 +23,23 @@ module ContainerRepository scope :with_verification_state, ->(state) { joins(:container_repository_state).where(container_repository_states: { verification_state: verification_state_value(state) }) } + # On primary, `verifiables` are records that can be checksummed and/or are replicable. + # On secondary, `verifiables` are records that have already been replicated + # and (ideally) have been checksummed on the primary + scope :verifiables, ->(primary_key_in = nil) do + node = ::GeoNode.current_node + + replicables = + available_replicables + .merge(object_storage_scope(node)) + + if ::Gitlab::Geo.org_mover_extend_selective_sync_to_primary_checksumming? + replicables.merge(selective_sync_scope(node, primary_key_in: primary_key_in, replicables: replicables)) + else + primary_key_in ? replicables.primary_key_in(primary_key_in) : replicables + end + end + def verification_state_object container_repository_state end @@ -42,6 +59,11 @@ def search(query) fuzzy_search(query, EE_SEARCHABLE_ATTRIBUTES) end + override :pluck_verifiable_ids_in_range + def pluck_verifiable_ids_in_range(range) + verifiables(range).pluck_primary_key + end + # @param primary_key_in [Range, ContainerRepository] arg to pass to primary_key_in scope # @return [ActiveRecord::Relation] everything that should be synced # to this node, restricted by primary key @@ -49,7 +71,12 @@ def search(query) def replicables_for_current_secondary(primary_key_in) return none unless replicator_class.replication_enabled? - super + node = ::Gitlab::Geo.current_node + + replicables = available_replicables.merge(object_storage_scope(node)) + + replicables + .merge(selective_sync_scope(node, primary_key_in: primary_key_in, replicables: replicables)) end override :verification_state_table_class @@ -59,10 +86,13 @@ def verification_state_table_class # @return [ActiveRecord::Relation] scope observing selective sync settings of the given node override :selective_sync_scope - def selective_sync_scope(node, **_params) - return all unless node.selective_sync? + def selective_sync_scope(node, **params) + replicables = params.fetch(:replicables, all) + replicables = replicables.primary_key_in(params[:primary_key_in]) if params[:primary_key_in].presence + + return replicables unless node.selective_sync? - project_id_in(::Project.selective_sync_scope(node)) + replicables.project_id_in(::Project.selective_sync_scope(node)) end end diff --git a/ee/spec/models/container_repository_spec.rb b/ee/spec/models/container_repository_spec.rb index d6dacd3288d4911ff7928690c2ae6f6e82645506..20e8ebbdb6c089c10fb95de822f1d4f7cb42f437 100644 --- a/ee/spec/models/container_repository_spec.rb +++ b/ee/spec/models/container_repository_spec.rb @@ -2,87 +2,7 @@ require 'spec_helper' -RSpec.describe ContainerRepository, feature_category: :geo_replication do - include_examples 'a verifiable model for verification state' do - let(:verifiable_model_record) { build(:container_repository) } - let(:unverifiable_model_record) { nil } - end - - describe '.replicables_for_current_secondary' do - let(:secondary) { create(:geo_node, :secondary) } - - let_it_be(:synced_group) { create(:group) } - let_it_be(:nested_group) { create(:group, parent: synced_group) } - let_it_be(:synced_project) { create(:project, group: synced_group) } - let_it_be(:synced_project_in_nested_group) { create(:project, group: nested_group) } - let_it_be(:unsynced_project) { create(:project) } - let_it_be(:project_broken_storage) { create(:project, :broken_storage) } - - let_it_be(:container_repository_1) { create(:container_repository, project: synced_project) } - let_it_be(:container_repository_2) { create(:container_repository, project: synced_project_in_nested_group) } - let_it_be(:container_repository_3) { create(:container_repository, project: unsynced_project) } - let_it_be(:container_repository_4) { create(:container_repository, project: project_broken_storage) } - - before do - stub_current_geo_node(secondary) - stub_registry_replication_config(enabled: true) - end - - context 'with registry replication disabled' do - before do - stub_registry_replication_config(enabled: false) - end - - it 'returns an empty relation' do - replicables = - described_class.replicables_for_current_secondary(described_class.minimum(:id)..described_class.maximum(:id)) - - expect(replicables).to be_empty - end - end - - context 'without selective sync' do - it 'returns all container repositories' do - expected = [container_repository_1, container_repository_2, container_repository_3, container_repository_4] - - replicables = - described_class.replicables_for_current_secondary(described_class.minimum(:id)..described_class.maximum(:id)) - - expect(replicables).to match_array(expected) - end - end - - context 'with selective sync by namespace' do - before do - secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) - end - - it 'excludes container repositories that are not in selectively synced projects' do - expected = [container_repository_1, container_repository_2] - - replicables = - described_class.replicables_for_current_secondary(described_class.minimum(:id)..described_class.maximum(:id)) - - expect(replicables).to match_array(expected) - end - end - - context 'with selective sync by shard' do - before do - secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - end - - it 'excludes container repositories that are not in selectively synced shards' do - expected = [container_repository_4] - - replicables = - described_class.replicables_for_current_secondary(described_class.minimum(:id)..described_class.maximum(:id)) - - expect(replicables).to match_array(expected) - end - end - end - +RSpec.describe ContainerRepository, feature_category: :container_registry do describe '.search' do let_it_be(:container_repository1) { create(:container_repository) } let_it_be(:container_repository2) { create(:container_repository) } @@ -126,6 +46,49 @@ end end + describe 'Geo', feature_category: :geo_replication do + describe 'associations' do + it 'has one verification state table class' do + is_expected + .to have_one(:container_repository_state) + .class_name('Geo::ContainerRepositoryState') + .inverse_of(:container_repository) + .autosave(false) + end + end + + include_examples 'a verifiable model for verification state' do + let(:verifiable_model_record) { build(:container_repository) } + let(:unverifiable_model_record) { nil } + end + + describe 'replication/verification' do + let_it_be(:group_1) { create(:group, organization: create(:organization)) } + let_it_be(:group_2) { create(:group, organization: create(:organization)) } + let_it_be(:nested_group_1) { create(:group, parent: group_1) } + let_it_be(:project_1) { create(:project, group: group_1) } + let_it_be(:project_2) { create(:project, group: nested_group_1) } + let_it_be(:project_3) { create(:project, :broken_storage, group: group_2) } + + # Container repository for the root group + let!(:first_replicable_and_in_selective_sync) do + create(:container_repository, project: project_1) + end + + # Container repository for a subgroup + let!(:second_replicable_and_in_selective_sync) do + create(:container_repository, project: project_2) + end + + # Container repository for a group not in selective sync (on broken storage shard) + let!(:last_replicable_and_not_in_selective_sync) do + create(:container_repository, project: project_3) + end + + include_examples 'Geo Framework selective sync behavior' + end + end + describe '#push_blob' do it "calls client's push blob with path passed" do gitlab_container_repository = create(:container_repository)