diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb index 078b05ff779185f6f53758085db4a7f62b2321ca..1d7c935aa95cfc6988c677472724b48d947b4489 100644 --- a/app/models/ci/secure_file.rb +++ b/app/models/ci/secure_file.rb @@ -24,6 +24,7 @@ class SecureFile < Ci::ApplicationRecord before_validation :assign_checksum scope :order_by_created_at, -> { order(created_at: :desc) } + scope :project_id_in, ->(ids) { where(project_id: ids) } default_value_for(:file_store) { Ci::SecureFileUploader.default_store } @@ -46,3 +47,5 @@ def generate_key_data end end end + +Ci::SecureFile.prepend_mod diff --git a/config/initializers_before_autoloader/000_inflections.rb b/config/initializers_before_autoloader/000_inflections.rb index 64686bdd9620050bea0dd5a6f9e7f7c5aa884ab9..70c9ec0a0bacf5ba84138425a2fa3cde133c27d0 100644 --- a/config/initializers_before_autoloader/000_inflections.rb +++ b/config/initializers_before_autoloader/000_inflections.rb @@ -15,6 +15,7 @@ inflect.uncountable %w( custom_emoji award_emoji + ci_secure_file_registry container_repository_registry design_registry event_log diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index a74f6c39b728f6dfcf83363ebe88b763645641d1..54dc877ead70a629527b75560672a36469477e9b 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -301,6 +301,16 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_uploads_verification_failed` | Gauge | 14.6 | Number of uploads verifications failed on secondary | `url` | | `gitlab_sli:rails_request_apdex:total` | Counter | 14.4 | The number of request-apdex measurements, [more information the development documentation](../../../development/application_slis/rails_request_apdex.md) | `endpoint_id`, `feature_category`, `request_urgency` | | `gitlab_sli:rails_request_apdex:success_total` | Counter | 14.4 | The number of successful requests that met the target duration for their urgency. Divide by `gitlab_sli:rails_requests_apdex:total` to get a success ratio | `endpoint_id`, `feature_category`, `request_urgency` | +| `geo_ci_secure_files` | Gauge | 15.3 | Number of secure files on primary | `url` | +| `geo_ci_secure_files_checksum_total` | Gauge | 15.3 | Number of secure files tried to checksum on primary | `url` | +| `geo_ci_secure_files_checksummed` | Gauge | 15.3 | Number of secure files successfully checksummed on primary | `url` | +| `geo_ci_secure_files_checksum_failed` | Gauge | 15.3 | Number of secure files failed to calculate the checksum on primary | `url` | +| `geo_ci_secure_files_synced` | Gauge | 15.3 | Number of syncable secure files synced on secondary | `url` | +| `geo_ci_secure_files_failed` | Gauge | 15.3 | Number of syncable secure files failed to sync on secondary | `url` | +| `geo_ci_secure_files_registry` | Gauge | 15.3 | Number of secure files in the registry | `url` | +| `geo_ci_secure_files_verification_total` | Gauge | 15.3 | Number of secure files verifications tried on secondary | `url` | +| `geo_ci_secure_files_verified` | Gauge | 15.3 | Number of secure files verified on secondary | `url` | +| `geo_ci_secure_files_verification_failed` | Gauge | 15.3 | Number of secure files verifications failed on secondary | `url` | ## Database load balancing metrics **(PREMIUM SELF)** diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index fbb583f5a5643323b8520b8c4cfea736e484c45e..b5b920ec0cd7b06eecf3c87a64ff21f1e0aa0c79 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -495,6 +495,19 @@ Example response: "job_artifacts_synced_in_percentage": "100.00%", "job_artifacts_verified_in_percentage": "100.00%", "job_artifacts_synced_missing_on_primary_count": 0, + "ci_secure_files_count": 5, + "ci_secure_files_checksum_total_count": 5, + "ci_secure_files_checksummed_count": 5, + "ci_secure_files_checksum_failed_count": 0, + "ci_secure_files_synced_count": 5, + "ci_secure_files_failed_count": 0, + "ci_secure_files_registry_count": 5, + "ci_secure_files_verification_total_count": 5, + "ci_secure_files_verified_count": 5, + "ci_secure_files_verification_failed_count": 0, + "ci_secure_files_synced_in_percentage": "100.00%", + "ci_secure_files_verified_in_percentage": "100.00%", + "ci_secure_files_synced_missing_on_primary_count": 0, }, { "geo_node_id": 2, @@ -830,6 +843,19 @@ Example response: "job_artifacts_synced_in_percentage": "100.00%", "job_artifacts_verified_in_percentage": "100.00%", "job_artifacts_synced_missing_on_primary_count": 0, + "ci_secure_files_count": 5, + "ci_secure_files_checksum_total_count": 5, + "ci_secure_files_checksummed_count": 5, + "ci_secure_files_checksum_failed_count": 0, + "ci_secure_files_synced_count": 5, + "ci_secure_files_failed_count": 0, + "ci_secure_files_registry_count": 5, + "ci_secure_files_verification_total_count": 5, + "ci_secure_files_verified_count": 5, + "ci_secure_files_verification_failed_count": 0, + "ci_secure_files_synced_in_percentage": "100.00%", + "ci_secure_files_verified_in_percentage": "100.00%", + "ci_secure_files_synced_missing_on_primary_count": 0, } ``` diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index fa317490c568779bf48a3ede0cbba46dd45a3eb6..e0d97cbc6fbfbbe541d5012ad241d258c3bcafad 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -6302,6 +6302,29 @@ The edge type for [`CiRunner`](#cirunner). | `node` | [`CiRunner`](#cirunner) | The item at the end of the edge. | | `webUrl` | [`String`](#string) | Web URL of the runner. The value depends on where you put this field in the query. You can use it for projects or groups. | +#### `CiSecureFileRegistryConnection` + +The connection type for [`CiSecureFileRegistry`](#cisecurefileregistry). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `edges` | [`[CiSecureFileRegistryEdge]`](#cisecurefileregistryedge) | A list of edges. | +| `nodes` | [`[CiSecureFileRegistry]`](#cisecurefileregistry) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +#### `CiSecureFileRegistryEdge` + +The edge type for [`CiSecureFileRegistry`](#cisecurefileregistry). + +##### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`CiSecureFileRegistry`](#cisecurefileregistry) | The item at the end of the edge. | + #### `CiStageConnection` The connection type for [`CiStage`](#cistage). @@ -10070,6 +10093,25 @@ Returns [`CiRunnerStatus!`](#cirunnerstatus). | ---- | ---- | ----------- | | `legacyMode` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.0. Will be removed in 17.0. In GitLab 16.0 and later, the field will act as if `legacyMode` is null. | +### `CiSecureFileRegistry` + +Represents the Geo replication and verification state of a ci_secure_file. + +#### Fields + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ciSecureFileId` | [`ID!`](#id) | ID of the Ci Secure File. | +| `createdAt` | [`Time`](#time) | Timestamp when the CiSecureFileRegistry was created. | +| `id` | [`ID!`](#id) | ID of the CiSecureFileRegistry. | +| `lastSyncFailure` | [`String`](#string) | Error message during sync of the CiSecureFileRegistry. | +| `lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the CiSecureFileRegistry. | +| `retryAt` | [`Time`](#time) | Timestamp after which the CiSecureFileRegistry is resynced. | +| `retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the CiSecureFileRegistry. | +| `state` | [`RegistryState`](#registrystate) | Sync state of the CiSecureFileRegistry. | +| `verificationRetryAt` | [`Time`](#time) | Timestamp after which the CiSecureFileRegistry is reverified. | +| `verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the CiSecureFileRegistry. | + ### `CiStage` #### Fields @@ -11681,6 +11723,22 @@ Represents an external issue. #### Fields with arguments +##### `GeoNode.ciSecureFileRegistries` + +Find Ci Secure File registries on this Geo node Available only when feature flag `geo_ci_secure_file_replication` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice. + +Returns [`CiSecureFileRegistryConnection`](#cisecurefileregistryconnection). + +This field returns a [connection](#connections). It accepts the +four standard [pagination arguments](#connection-pagination-arguments): +`before: String`, `after: String`, `first: Int`, `last: Int`. + +###### Arguments + +| Name | Type | Description | +| ---- | ---- | ----------- | +| `ids` | [`[ID!]`](#id) | Filters registries by their ID. | + ##### `GeoNode.groupWikiRepositoryRegistries` Find group wiki repository registries on this Geo node. diff --git a/ee/app/finders/geo/ci_secure_file_registry_finder.rb b/ee/app/finders/geo/ci_secure_file_registry_finder.rb new file mode 100644 index 0000000000000000000000000000000000000000..ef4f69a9425946cdc2ded77d8276d237647ba4d9 --- /dev/null +++ b/ee/app/finders/geo/ci_secure_file_registry_finder.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Geo + class CiSecureFileRegistryFinder + include FrameworkRegistryFinder + end +end diff --git a/ee/app/graphql/resolvers/geo/ci_secure_file_registries_resolver.rb b/ee/app/graphql/resolvers/geo/ci_secure_file_registries_resolver.rb new file mode 100644 index 0000000000000000000000000000000000000000..2818b4be0efaaff5d1b5f1cb319db8cfdbc93b95 --- /dev/null +++ b/ee/app/graphql/resolvers/geo/ci_secure_file_registries_resolver.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Resolvers + module Geo + class CiSecureFileRegistriesResolver < BaseResolver + type ::Types::Geo::GeoNodeType.connection_type, null: true + + include RegistriesResolver + end + end +end diff --git a/ee/app/graphql/types/geo/ci_secure_file_registry_type.rb b/ee/app/graphql/types/geo/ci_secure_file_registry_type.rb new file mode 100644 index 0000000000000000000000000000000000000000..4aae738baa6f431d9421f31cd90d80f792be7423 --- /dev/null +++ b/ee/app/graphql/types/geo/ci_secure_file_registry_type.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Types + module Geo + # rubocop:disable Graphql/AuthorizeTypes because it is included + class CiSecureFileRegistryType < BaseObject + graphql_name 'CiSecureFileRegistry' + description 'Represents the Geo replication and verification state of a ci_secure_file.' + + include ::Types::Geo::RegistryType + + field :ci_secure_file_id, GraphQL::Types::ID, null: false, description: 'ID of the Ci Secure File.' + end + end +end diff --git a/ee/app/graphql/types/geo/geo_node_type.rb b/ee/app/graphql/types/geo/geo_node_type.rb index 314ed1b2b5e003620d9287b7f673a0ebbbcd314d..a0be29b5bfe504c3e87f1cbf1c4508bc2fa6bcff 100644 --- a/ee/app/graphql/types/geo/geo_node_type.rb +++ b/ee/app/graphql/types/geo/geo_node_type.rb @@ -7,6 +7,11 @@ class GeoNodeType < BaseObject authorize :read_geo_node + field :ci_secure_file_registries, ::Types::Geo::CiSecureFileRegistryType.connection_type, + null: true, + resolver: ::Resolvers::Geo::CiSecureFileRegistriesResolver, + description: 'Find Ci Secure File registries on this Geo node', + feature_flag: :geo_ci_secure_file_replication field :container_repositories_max_capacity, GraphQL::Types::Int, null: true, description: 'Maximum concurrency of container repository sync for this secondary node.' field :enabled, GraphQL::Types::Boolean, null: true, description: 'Indicates whether this Geo node is enabled.' field :files_max_capacity, GraphQL::Types::Int, null: true, description: 'Maximum concurrency of LFS/attachment backfill for this secondary node.' diff --git a/ee/app/models/concerns/ee/ci/artifactable.rb b/ee/app/models/concerns/ee/ci/artifactable.rb index 193649891033e8c82d8cc71a56694e974b3af759..e87d66ed713cae0542d840f0ba6c9216a3028ecf 100644 --- a/ee/app/models/concerns/ee/ci/artifactable.rb +++ b/ee/app/models/concerns/ee/ci/artifactable.rb @@ -6,8 +6,8 @@ module Artifactable extend ActiveSupport::Concern class_methods do - # @param primary_key_in [Range, Ci::{Pipeline|Job}Artifact] arg to pass to primary_key_in scope - # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key + # @param primary_key_in [Range, Ci::{PipelineArtifact|JobArtifact|SecureFile}] arg to pass to primary_key_in scope + # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key def replicables_for_current_secondary(primary_key_in) node = ::Gitlab::Geo.current_node @@ -18,7 +18,7 @@ def replicables_for_current_secondary(primary_key_in) selective_sync_scope(node, replicables) end - # @return [ActiveRecord::Relation] observing object storage settings of the given node + # @return [ActiveRecord::Relation] observing object storage settings of the given node def object_storage_scope(node) return all if node.sync_object_storage? @@ -28,7 +28,7 @@ def object_storage_scope(node) # The primary_key_in in replicables_for_current_secondary method is at most a range of IDs with a maximum of 10_000 records # between them. We can additionally reduce the batch size to 1_000 just for pipeline artifacts and job artifacts if needed. # - # @return [ActiveRecord::Relation] observing selective sync settings of the given node + # @return [ActiveRecord::Relation] observing selective sync settings of the given node def selective_sync_scope(node, replicables) return replicables unless node.selective_sync? diff --git a/ee/app/models/ee/ci/secure_file.rb b/ee/app/models/ee/ci/secure_file.rb new file mode 100644 index 0000000000000000000000000000000000000000..d2bb21e94600f66be02fb3a5c5a06c79ac5cb4a5 --- /dev/null +++ b/ee/app/models/ee/ci/secure_file.rb @@ -0,0 +1,66 @@ +# frozen_string_literal: true + +module EE + # CI::SecureFile EE mixin + # + # This module is intended to encapsulate EE-specific model logic + # and be prepended in the `Ci::SecureFile` model + module Ci + module SecureFile + extend ActiveSupport::Concern + + prepended do + include ::Geo::ReplicableModel + include ::Geo::VerifiableModel + include Artifactable + + delegate(*::Geo::VerificationState::VERIFICATION_METHODS, to: :ci_secure_file_state) + + with_replicator Geo::CiSecureFileReplicator + + has_one :ci_secure_file_state, autosave: false, inverse_of: :ci_secure_file, + class_name: 'Geo::CiSecureFileState', foreign_key: :ci_secure_file_id + + after_save :save_verification_details + + scope :with_verification_state, ->(state) { + joins(:ci_secure_file_state).where( + ci_secure_file_states: { + verification_state: verification_state_value(state) + } + ) + } + scope :checksummed, -> { + joins(:ci_secure_file_state).where.not( + ci_secure_file_states: { verification_checksum: nil } + ) + } + scope :not_checksummed, -> { + joins(:ci_secure_file_state).where( + ci_secure_file_states: { verification_checksum: nil } + ) + } + + scope :available_verifiables, -> { joins(:ci_secure_file_state) } + scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) } + + def verification_state_object + ci_secure_file_state + end + end + + class_methods do + extend ::Gitlab::Utils::Override + + override :verification_state_table_class + def verification_state_table_class + ::Geo::CiSecureFileState + end + end + + def ci_secure_file_state + super || build_ci_secure_file_state + end + end + end +end diff --git a/ee/app/models/geo/ci_secure_file_registry.rb b/ee/app/models/geo/ci_secure_file_registry.rb new file mode 100644 index 0000000000000000000000000000000000000000..6d4e4255ed9768a8191d11b150015770210b15d8 --- /dev/null +++ b/ee/app/models/geo/ci_secure_file_registry.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Geo + class CiSecureFileRegistry < Geo::BaseRegistry + include ::Geo::ReplicableRegistry + include ::Geo::VerifiableRegistry + + MODEL_CLASS = ::Ci::SecureFile + MODEL_FOREIGN_KEY = :ci_secure_file_id + + belongs_to :ci_secure_file, class_name: 'Ci::SecureFile' + end +end diff --git a/ee/app/models/geo/ci_secure_file_state.rb b/ee/app/models/geo/ci_secure_file_state.rb new file mode 100644 index 0000000000000000000000000000000000000000..247fa5d084374c5ded107522930fa51d6383cfc5 --- /dev/null +++ b/ee/app/models/geo/ci_secure_file_state.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true +module Geo + class CiSecureFileState < Ci::ApplicationRecord + include EachBatch + include ::Geo::VerificationStateDefinition + + self.primary_key = :ci_secure_file_id + self.table_name = :ci_secure_file_states + + belongs_to :ci_secure_file, inverse_of: :ci_secure_file_state, class_name: 'Ci::SecureFile' + + validates :verification_failure, length: { maximum: 255 } + validates :verification_state, :ci_secure_file, presence: true + end +end diff --git a/ee/app/replicators/geo/ci_secure_file_replicator.rb b/ee/app/replicators/geo/ci_secure_file_replicator.rb new file mode 100644 index 0000000000000000000000000000000000000000..b13e22f0e8aae6ee8b1b1aba1a19e6a7386d5f95 --- /dev/null +++ b/ee/app/replicators/geo/ci_secure_file_replicator.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Geo + class CiSecureFileReplicator < Gitlab::Geo::Replicator + include ::Geo::BlobReplicatorStrategy + extend ::Gitlab::Utils::Override + + def self.model + ::Ci::SecureFile + end + + def carrierwave_uploader + model_record.file + end + + override :verification_feature_flag_enabled? + def self.verification_feature_flag_enabled? + # We are adding verification at the same time as replication, so we + # don't need to toggle verification separately from replication. When + # the replication feature flag is off, then verification is also off + # (see `VerifiableReplicator.verification_enabled?`) + true + end + end +end diff --git a/ee/app/workers/geo/secondary/registry_consistency_worker.rb b/ee/app/workers/geo/secondary/registry_consistency_worker.rb index 10c908caa2c762666d584bda56e9e96b52849f1d..3eebaf2f74c2528e7775e4c3926ea06a1932a05a 100644 --- a/ee/app/workers/geo/secondary/registry_consistency_worker.rb +++ b/ee/app/workers/geo/secondary/registry_consistency_worker.rb @@ -31,7 +31,8 @@ class RegistryConsistencyWorker Geo::UploadRegistry, Geo::SnippetRepositoryRegistry, Geo::GroupWikiRepositoryRegistry, - Geo::PagesDeploymentRegistry + Geo::PagesDeploymentRegistry, + Geo::CiSecureFileRegistry ].freeze BATCH_SIZE = 10000 diff --git a/ee/config/feature_flags/development/geo_ci_secure_file_replication.yml b/ee/config/feature_flags/development/geo_ci_secure_file_replication.yml new file mode 100644 index 0000000000000000000000000000000000000000..db3b2ba9dd722b04368f524b072715cd7d70ee7b --- /dev/null +++ b/ee/config/feature_flags/development/geo_ci_secure_file_replication.yml @@ -0,0 +1,8 @@ +--- +name: geo_ci_secure_file_replication +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430 +rollout_issue_url: +milestone: '15.2' +type: development +group: group::incubation +default_enabled: false diff --git a/ee/lib/gitlab/geo.rb b/ee/lib/gitlab/geo.rb index eb0485d17fa41ef6fe716bdc8aa50c5016b75cb5..7de3591fbbf1cafe07f975dc328fea64dc593c1b 100644 --- a/ee/lib/gitlab/geo.rb +++ b/ee/lib/gitlab/geo.rb @@ -32,7 +32,8 @@ module Geo ::Geo::PipelineArtifactReplicator, ::Geo::PagesDeploymentReplicator, ::Geo::UploadReplicator, - ::Geo::JobArtifactReplicator + ::Geo::JobArtifactReplicator, + ::Geo::CiSecureFileReplicator ].freeze # We "regenerate" an 1hour valid JWT every 30 minutes, resulting in diff --git a/ee/spec/factories/ci/secure_files.rb b/ee/spec/factories/ci/secure_files.rb new file mode 100644 index 0000000000000000000000000000000000000000..629350e35470567a743e3e123856468ebe5e134b --- /dev/null +++ b/ee/spec/factories/ci/secure_files.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :ee_ci_secure_file, class: '::Ci::SecureFile', parent: :ci_secure_file do + trait(:verification_succeeded) do + with_file + verification_checksum { 'abc' } + verification_state { Ci::SecureFile.verification_state_value(:verification_succeeded) } + end + + trait(:verification_failed) do + with_file + verification_failure { 'Could not calculate the checksum' } + verification_state { Ci::SecureFile.verification_state_value(:verification_failed) } + end + end +end diff --git a/ee/spec/factories/geo/ci_secure_file_registry.rb b/ee/spec/factories/geo/ci_secure_file_registry.rb new file mode 100644 index 0000000000000000000000000000000000000000..a17445b7f06cb27e8c7177df8bd7eb166ab784fd --- /dev/null +++ b/ee/spec/factories/geo/ci_secure_file_registry.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :geo_ci_secure_file_registry, class: 'Geo::CiSecureFileRegistry' do + ci_secure_file # This association should have data, like a file or repository + state { Geo::CiSecureFileRegistry.state_value(:pending) } + + trait :synced do + state { Geo::CiSecureFileRegistry.state_value(:synced) } + last_synced_at { 5.days.ago } + end + + trait :failed do + state { Geo::CiSecureFileRegistry.state_value(:failed) } + last_synced_at { 1.day.ago } + retry_count { 2 } + last_sync_failure { 'Random error' } + end + + trait :started do + state { Geo::CiSecureFileRegistry.state_value(:started) } + last_synced_at { 1.day.ago } + retry_count { 0 } + end + + trait :verification_succeeded do + verification_checksum { 'e079a831cab27bcda7d81cd9b48296d0c3dd92ef' } + verification_state { Geo::CiSecureFileRegistry.verification_state_value(:verification_succeeded) } + verified_at { 5.days.ago } + end + end +end diff --git a/ee/spec/factories/geo/ci_secure_file_states.rb b/ee/spec/factories/geo/ci_secure_file_states.rb new file mode 100644 index 0000000000000000000000000000000000000000..a390bd837ec9b921a98b6fab00957ad2148c9520 --- /dev/null +++ b/ee/spec/factories/geo/ci_secure_file_states.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :geo_ci_secure_file_state, class: 'Geo::CiSecureFileState' do + ci_secure_file + + trait(:checksummed) do + verification_checksum { 'abc' } + end + + trait(:checksum_failure) do + verification_failure { 'Could not calculate the checksum' } + end + end +end diff --git a/ee/spec/finders/geo/ci_secure_file_registry_finder_spec.rb b/ee/spec/finders/geo/ci_secure_file_registry_finder_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..764c29c5d591338c42aed4b1969025035654454a --- /dev/null +++ b/ee/spec/finders/geo/ci_secure_file_registry_finder_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::CiSecureFileRegistryFinder do + it_behaves_like 'a framework registry finder', :geo_ci_secure_file_registry +end diff --git a/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json b/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json index 50284970513a1ee516fa76168f3666c1128b5bc6..213c4311da0743d73cc3d2d4fc5abdff8d19eeb1 100644 --- a/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json +++ b/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json @@ -29,6 +29,18 @@ "job_artifacts_verified_count", "job_artifacts_verification_failed_count", "job_artifacts_verified_in_percentage", + "ci_secure_files_count", + "ci_secure_files_checksum_total_count", + "ci_secure_files_checksummed_count", + "ci_secure_files_checksum_failed_count", + "ci_secure_files_synced_count", + "ci_secure_files_failed_count", + "ci_secure_files_registry_count", + "ci_secure_files_verification_total_count", + "ci_secure_files_verified_count", + "ci_secure_files_verification_failed_count", + "ci_secure_files_synced_in_percentage", + "ci_secure_files_verified_in_percentage", "db_replication_lag_seconds", "container_repositories_replication_enabled", "container_repositories_count", @@ -219,6 +231,18 @@ "job_artifacts_verified_count": { "type": ["integer", "null"] }, "job_artifacts_verification_failed_count": { "type": ["integer", "null"] }, "job_artifacts_verified_in_percentage": { "type": "string" }, + "ci_secure_files_count": { "type": "integer" }, + "ci_secure_files_failed_count": { "type": ["integer", "null"] }, + "ci_secure_files_synced_count": { "type": ["integer", "null"] }, + "ci_secure_files_synced_in_percentage": { "type": "string" }, + "ci_secure_files_checksum_total_count": { "type": ["integer", "null"] }, + "ci_secure_files_checksummed_count": { "type": ["integer", "null"] }, + "ci_secure_files_checksum_failed_count": { "type": ["integer", "null"] }, + "ci_secure_files_registry_count": { "type": ["integer", "null"] }, + "ci_secure_files_verification_total_count": { "type": ["integer", "null"] }, + "ci_secure_files_verified_count": { "type": ["integer", "null"] }, + "ci_secure_files_verification_failed_count": { "type": ["integer", "null"] }, + "ci_secure_files_verified_in_percentage": { "type": "string" }, "container_repositories_replication_enabled": { "type": ["boolean", "null"] }, "container_repositories_count": { "type": "integer" }, "container_repositories_failed_count": { "type": ["integer", "null"] }, diff --git a/ee/spec/graphql/resolvers/geo/ci_secure_file_registries_resolver_spec.rb b/ee/spec/graphql/resolvers/geo/ci_secure_file_registries_resolver_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..2a7c26a68d052ec4eafb9d075d1836e57b47803a --- /dev/null +++ b/ee/spec/graphql/resolvers/geo/ci_secure_file_registries_resolver_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Resolvers::Geo::CiSecureFileRegistriesResolver do + it_behaves_like 'a Geo registries resolver', :geo_ci_secure_file_registry +end diff --git a/ee/spec/graphql/types/geo/ci_secure_file_registry_type_spec.rb b/ee/spec/graphql/types/geo/ci_secure_file_registry_type_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..2c80c7463fd29b7deb80700311ad8a24a864ac06 --- /dev/null +++ b/ee/spec/graphql/types/geo/ci_secure_file_registry_type_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe GitlabSchema.types['CiSecureFileRegistry'] do + it_behaves_like 'a Geo registry type' + + it 'has the expected fields (other than those included in RegistryType)' do + expected_fields = %i[ci_secure_file_id] + + expect(described_class).to have_graphql_fields(*expected_fields).at_least + end +end diff --git a/ee/spec/graphql/types/geo/geo_node_type_spec.rb b/ee/spec/graphql/types/geo/geo_node_type_spec.rb index 6d9becaa901c083d8bc4da165fb80c6c2bfabcca..676251819e0cb5f9efaabb06cd41db3d0076068e 100644 --- a/ee/spec/graphql/types/geo/geo_node_type_spec.rb +++ b/ee/spec/graphql/types/geo/geo_node_type_spec.rb @@ -15,7 +15,7 @@ package_file_registries snippet_repository_registries terraform_state_version_registries group_wiki_repository_registries pages_deployment_registries lfs_object_registries pipeline_artifact_registries - upload_registries job_artifact_registries + upload_registries job_artifact_registries ci_secure_file_registries ] expect(described_class).to have_graphql_fields(*expected_fields) diff --git a/ee/spec/models/ee/ci/secure_file_spec.rb b/ee/spec/models/ee/ci/secure_file_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..5f11c315311136ddc460bf528f32057b39df2f5d --- /dev/null +++ b/ee/spec/models/ee/ci/secure_file_spec.rb @@ -0,0 +1,89 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::SecureFile do + using RSpec::Parameterized::TableSyntax + include EE::GeoHelpers + + include_examples 'a replicable model with a separate table for verification state' do + let(:project) { create(:project) } + let(:verifiable_model_record) { build(:ci_secure_file, project: project) } + + let(:unverifiable_model_record) do + stub_ci_secure_file_object_storage + file = build(:ci_secure_file, :remote_store, project: project) + stub_ci_secure_file_object_storage(enabled: false) + + file + end + end + + describe '#replicables_for_current_secondary' do + # Selective sync is configured relative to the secure file's project. + # + # Permutations of sync_object_storage combined with object-stored-artifacts + # are tested in code, because the logic is simple, and to do it in the table + # would quadruple its size and have too much duplication. + where(:selective_sync_namespaces, :selective_sync_shards, :factory, :project_factory, :include_expectation) do + nil | nil | [:ci_secure_file] | [:project] | true + # selective sync by shard + nil | :model | [:ci_secure_file] | [:project] | true + nil | :other | [:ci_secure_file] | [:project] | false + # selective sync by namespace + :model_parent | nil | [:ci_secure_file] | [:project] | true + :model_parent_parent | nil | [:ci_secure_file] | [:project, :in_subgroup] | true + :other | nil | [:ci_secure_file] | [:project] | false + :other | nil | [:ci_secure_file] | [:project, :in_subgroup] | false + end + + with_them do + subject(:ci_secure_file_included) { described_class.replicables_for_current_secondary(ci_secure_file).exists? } + + let(:project) { create(*project_factory) } # rubocop:disable Rails/SaveBang + let(:node) do + create(:geo_node_with_selective_sync_for, + model: project, + namespaces: selective_sync_namespaces, + shards: selective_sync_shards, + sync_object_storage: sync_object_storage) + end + + before do + stub_current_geo_node(node) + end + + context 'when sync object storage is enabled' do + let(:sync_object_storage) { true } + + context 'when the ci secure file is locally stored' do + let(:ci_secure_file) { create(*factory, project: project) } + + it { is_expected.to eq(include_expectation) } + end + + context 'when the ci secure file is object stored' do + let(:ci_secure_file) { create(*factory, :remote_store, project: project) } + + it { is_expected.to eq(include_expectation) } + end + end + + context 'when sync object storage is disabled' do + let(:sync_object_storage) { false } + + context 'when the ci secure file is locally stored' do + let(:ci_secure_file) { create(*factory, project: project) } + + it { is_expected.to eq(include_expectation) } + end + + context 'when the ci secure file is object stored' do + let(:ci_secure_file) { create(*factory, :remote_store, project: project) } + + it { is_expected.to be_falsey } + end + end + end + end +end diff --git a/ee/spec/models/geo/ci_secure_file_registry_spec.rb b/ee/spec/models/geo/ci_secure_file_registry_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..1c31948f25add9d1b3a531d8f4f9a81fda9cc668 --- /dev/null +++ b/ee/spec/models/geo/ci_secure_file_registry_spec.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::CiSecureFileRegistry, :geo, type: :model do + let_it_be(:registry) { create(:geo_ci_secure_file_registry) } + + specify 'factory is valid' do + expect(registry).to be_valid + end + + include_examples 'a Geo framework registry' + include_examples 'a Geo verifiable registry' +end diff --git a/ee/spec/models/geo_node_status_spec.rb b/ee/spec/models/geo_node_status_spec.rb index ae7b641c0c8b42339ada359f8bd6be62e9c6b54c..22cab9fb08a70bf88751f7f36ba6a5eb43d44769 100644 --- a/ee/spec/models/geo_node_status_spec.rb +++ b/ee/spec/models/geo_node_status_spec.rb @@ -1031,6 +1031,7 @@ Geo::PagesDeploymentReplicator | :pages_deployment | :geo_pages_deployment_registry Geo::UploadReplicator | :upload | :geo_upload_registry Geo::JobArtifactReplicator | :ci_job_artifact | :geo_job_artifact_registry + Geo::CiSecureFileReplicator | :ci_secure_file | :geo_ci_secure_file_registry end with_them do diff --git a/ee/spec/replicators/geo/ci_secure_file_replicator_spec.rb b/ee/spec/replicators/geo/ci_secure_file_replicator_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..15fcefdfe2ff8c4fa1eda9b646fd3aa2acf423ac --- /dev/null +++ b/ee/spec/replicators/geo/ci_secure_file_replicator_spec.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::CiSecureFileReplicator do + let(:project) { create(:project) } + let(:model_record) { create(:ci_secure_file, project: project) } + + include_examples 'a blob replicator' + include_examples 'a verifiable replicator' +end diff --git a/ee/spec/requests/api/graphql/geo/registries_spec.rb b/ee/spec/requests/api/graphql/geo/registries_spec.rb index 7c4cdd0680e6d5dfe54931b2bf27274f2be7b234..7c00649dedf0dacea5a83eb4a46a0c3b75434dbf 100644 --- a/ee/spec/requests/api/graphql/geo/registries_spec.rb +++ b/ee/spec/requests/api/graphql/geo/registries_spec.rb @@ -65,4 +65,11 @@ registry_factory: :geo_job_artifact_registry, registry_foreign_key_field_name: 'artifactId' } + + it_behaves_like 'gets registries for', { + field_name: 'ciSecureFileRegistries', + registry_class_name: 'CiSecureFileRegistry', + registry_factory: :geo_ci_secure_file_registry, + registry_foreign_key_field_name: 'ciSecureFileId' + } end diff --git a/ee/spec/services/geo/registry_consistency_service_spec.rb b/ee/spec/services/geo/registry_consistency_service_spec.rb index d182ee2bb1d625f16ea94c291eb980b48734a8c2..187ed26946c572bee6c154f9129bc52a9970377d 100644 --- a/ee/spec/services/geo/registry_consistency_service_spec.rb +++ b/ee/spec/services/geo/registry_consistency_service_spec.rb @@ -21,7 +21,8 @@ def model_class_factory_name(registry_class) Geo::MergeRequestDiffRegistry => :external_merge_request_diff, Geo::PackageFileRegistry => :package_file, Geo::UploadRegistry => :upload, - Geo::JobArtifactRegistry => :ci_job_artifact + Geo::JobArtifactRegistry => :ci_job_artifact, + Geo::CiSecureFileRegistry => :ci_secure_file }.fetch(registry_class, default_factory_name) end diff --git a/ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb b/ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb index 94db718fda30c343bfaa1e4d9963e27e887b01c4..22ca629676c0856bbed29b8ab90682d328155421 100644 --- a/ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb +++ b/ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb @@ -88,6 +88,7 @@ pipeline_artifact = create(:ci_pipeline_artifact) upload = create(:upload) pages_deployment = create(:pages_deployment) + ci_secure_file = create(:ci_secure_file) expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(0) expect(Geo::DesignRegistry.where(project_id: project.id).count).to eq(0) @@ -101,6 +102,7 @@ expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(0) expect(Geo::PagesDeploymentRegistry.where(pages_deployment: pages_deployment.id).count).to eq(0) expect(Geo::JobArtifactRegistry.where(job_artifact: job_artifact.id).count).to eq(0) + expect(Geo::CiSecureFileRegistry.where(ci_secure_file: ci_secure_file.id).count).to eq(0) subject.perform @@ -116,6 +118,7 @@ expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(1) expect(Geo::PagesDeploymentRegistry.where(pages_deployment: pages_deployment.id).count).to eq(1) expect(Geo::JobArtifactRegistry.where(job_artifact: job_artifact.id).count).to eq(1) + expect(Geo::CiSecureFileRegistry.where(ci_secure_file: ci_secure_file.id).count).to eq(1) end context 'when the current Geo node is disabled or primary' do diff --git a/spec/factories/ci/secure_files.rb b/spec/factories/ci/secure_files.rb index 9afec5db85879aec839cdac0b74d5b6c9dec2b6c..74988202c71477a17e54ac79b1cbfec843dd4d0a 100644 --- a/spec/factories/ci/secure_files.rb +++ b/spec/factories/ci/secure_files.rb @@ -6,5 +6,11 @@ file { fixture_file_upload('spec/fixtures/ci_secure_files/upload-keystore.jks', 'application/octet-stream') } checksum { 'foo1234' } project + + trait :remote_store do + after(:create) do |ci_secure_file| + ci_secure_file.update!(file_store: ObjectStorage::Store::REMOTE) + end + end end end