diff --git a/app/models/ci/secure_file.rb b/app/models/ci/secure_file.rb
index 078b05ff779185f6f53758085db4a7f62b2321ca..1d7c935aa95cfc6988c677472724b48d947b4489 100644
--- a/app/models/ci/secure_file.rb
+++ b/app/models/ci/secure_file.rb
@@ -24,6 +24,7 @@ class SecureFile < Ci::ApplicationRecord
before_validation :assign_checksum
scope :order_by_created_at, -> { order(created_at: :desc) }
+ scope :project_id_in, ->(ids) { where(project_id: ids) }
default_value_for(:file_store) { Ci::SecureFileUploader.default_store }
@@ -46,3 +47,5 @@ def generate_key_data
end
end
end
+
+Ci::SecureFile.prepend_mod
diff --git a/config/initializers_before_autoloader/000_inflections.rb b/config/initializers_before_autoloader/000_inflections.rb
index 64686bdd9620050bea0dd5a6f9e7f7c5aa884ab9..70c9ec0a0bacf5ba84138425a2fa3cde133c27d0 100644
--- a/config/initializers_before_autoloader/000_inflections.rb
+++ b/config/initializers_before_autoloader/000_inflections.rb
@@ -15,6 +15,7 @@
inflect.uncountable %w(
custom_emoji
award_emoji
+ ci_secure_file_registry
container_repository_registry
design_registry
event_log
diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md
index a74f6c39b728f6dfcf83363ebe88b763645641d1..54dc877ead70a629527b75560672a36469477e9b 100644
--- a/doc/administration/monitoring/prometheus/gitlab_metrics.md
+++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md
@@ -301,6 +301,16 @@ configuration option in `gitlab.yml`. These metrics are served from the
| `geo_uploads_verification_failed` | Gauge | 14.6 | Number of uploads verifications failed on secondary | `url` |
| `gitlab_sli:rails_request_apdex:total` | Counter | 14.4 | The number of request-apdex measurements, [more information the development documentation](../../../development/application_slis/rails_request_apdex.md) | `endpoint_id`, `feature_category`, `request_urgency` |
| `gitlab_sli:rails_request_apdex:success_total` | Counter | 14.4 | The number of successful requests that met the target duration for their urgency. Divide by `gitlab_sli:rails_requests_apdex:total` to get a success ratio | `endpoint_id`, `feature_category`, `request_urgency` |
+| `geo_ci_secure_files` | Gauge | 15.3 | Number of secure files on primary | `url` |
+| `geo_ci_secure_files_checksum_total` | Gauge | 15.3 | Number of secure files tried to checksum on primary | `url` |
+| `geo_ci_secure_files_checksummed` | Gauge | 15.3 | Number of secure files successfully checksummed on primary | `url` |
+| `geo_ci_secure_files_checksum_failed` | Gauge | 15.3 | Number of secure files failed to calculate the checksum on primary | `url` |
+| `geo_ci_secure_files_synced` | Gauge | 15.3 | Number of syncable secure files synced on secondary | `url` |
+| `geo_ci_secure_files_failed` | Gauge | 15.3 | Number of syncable secure files failed to sync on secondary | `url` |
+| `geo_ci_secure_files_registry` | Gauge | 15.3 | Number of secure files in the registry | `url` |
+| `geo_ci_secure_files_verification_total` | Gauge | 15.3 | Number of secure files verifications tried on secondary | `url` |
+| `geo_ci_secure_files_verified` | Gauge | 15.3 | Number of secure files verified on secondary | `url` |
+| `geo_ci_secure_files_verification_failed` | Gauge | 15.3 | Number of secure files verifications failed on secondary | `url` |
## Database load balancing metrics **(PREMIUM SELF)**
diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md
index fbb583f5a5643323b8520b8c4cfea736e484c45e..b5b920ec0cd7b06eecf3c87a64ff21f1e0aa0c79 100644
--- a/doc/api/geo_nodes.md
+++ b/doc/api/geo_nodes.md
@@ -495,6 +495,19 @@ Example response:
"job_artifacts_synced_in_percentage": "100.00%",
"job_artifacts_verified_in_percentage": "100.00%",
"job_artifacts_synced_missing_on_primary_count": 0,
+ "ci_secure_files_count": 5,
+ "ci_secure_files_checksum_total_count": 5,
+ "ci_secure_files_checksummed_count": 5,
+ "ci_secure_files_checksum_failed_count": 0,
+ "ci_secure_files_synced_count": 5,
+ "ci_secure_files_failed_count": 0,
+ "ci_secure_files_registry_count": 5,
+ "ci_secure_files_verification_total_count": 5,
+ "ci_secure_files_verified_count": 5,
+ "ci_secure_files_verification_failed_count": 0,
+ "ci_secure_files_synced_in_percentage": "100.00%",
+ "ci_secure_files_verified_in_percentage": "100.00%",
+ "ci_secure_files_synced_missing_on_primary_count": 0,
},
{
"geo_node_id": 2,
@@ -830,6 +843,19 @@ Example response:
"job_artifacts_synced_in_percentage": "100.00%",
"job_artifacts_verified_in_percentage": "100.00%",
"job_artifacts_synced_missing_on_primary_count": 0,
+ "ci_secure_files_count": 5,
+ "ci_secure_files_checksum_total_count": 5,
+ "ci_secure_files_checksummed_count": 5,
+ "ci_secure_files_checksum_failed_count": 0,
+ "ci_secure_files_synced_count": 5,
+ "ci_secure_files_failed_count": 0,
+ "ci_secure_files_registry_count": 5,
+ "ci_secure_files_verification_total_count": 5,
+ "ci_secure_files_verified_count": 5,
+ "ci_secure_files_verification_failed_count": 0,
+ "ci_secure_files_synced_in_percentage": "100.00%",
+ "ci_secure_files_verified_in_percentage": "100.00%",
+ "ci_secure_files_synced_missing_on_primary_count": 0,
}
```
diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md
index fa317490c568779bf48a3ede0cbba46dd45a3eb6..e0d97cbc6fbfbbe541d5012ad241d258c3bcafad 100644
--- a/doc/api/graphql/reference/index.md
+++ b/doc/api/graphql/reference/index.md
@@ -6302,6 +6302,29 @@ The edge type for [`CiRunner`](#cirunner).
| `node` | [`CiRunner`](#cirunner) | The item at the end of the edge. |
| `webUrl` | [`String`](#string) | Web URL of the runner. The value depends on where you put this field in the query. You can use it for projects or groups. |
+#### `CiSecureFileRegistryConnection`
+
+The connection type for [`CiSecureFileRegistry`](#cisecurefileregistry).
+
+##### Fields
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| `edges` | [`[CiSecureFileRegistryEdge]`](#cisecurefileregistryedge) | A list of edges. |
+| `nodes` | [`[CiSecureFileRegistry]`](#cisecurefileregistry) | A list of nodes. |
+| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. |
+
+#### `CiSecureFileRegistryEdge`
+
+The edge type for [`CiSecureFileRegistry`](#cisecurefileregistry).
+
+##### Fields
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| `cursor` | [`String!`](#string) | A cursor for use in pagination. |
+| `node` | [`CiSecureFileRegistry`](#cisecurefileregistry) | The item at the end of the edge. |
+
#### `CiStageConnection`
The connection type for [`CiStage`](#cistage).
@@ -10070,6 +10093,25 @@ Returns [`CiRunnerStatus!`](#cirunnerstatus).
| ---- | ---- | ----------- |
| `legacyMode` **{warning-solid}** | [`String`](#string) | **Deprecated** in 15.0. Will be removed in 17.0. In GitLab 16.0 and later, the field will act as if `legacyMode` is null. |
+### `CiSecureFileRegistry`
+
+Represents the Geo replication and verification state of a ci_secure_file.
+
+#### Fields
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| `ciSecureFileId` | [`ID!`](#id) | ID of the Ci Secure File. |
+| `createdAt` | [`Time`](#time) | Timestamp when the CiSecureFileRegistry was created. |
+| `id` | [`ID!`](#id) | ID of the CiSecureFileRegistry. |
+| `lastSyncFailure` | [`String`](#string) | Error message during sync of the CiSecureFileRegistry. |
+| `lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the CiSecureFileRegistry. |
+| `retryAt` | [`Time`](#time) | Timestamp after which the CiSecureFileRegistry is resynced. |
+| `retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the CiSecureFileRegistry. |
+| `state` | [`RegistryState`](#registrystate) | Sync state of the CiSecureFileRegistry. |
+| `verificationRetryAt` | [`Time`](#time) | Timestamp after which the CiSecureFileRegistry is reverified. |
+| `verifiedAt` | [`Time`](#time) | Timestamp of the most recent successful verification of the CiSecureFileRegistry. |
+
### `CiStage`
#### Fields
@@ -11681,6 +11723,22 @@ Represents an external issue.
#### Fields with arguments
+##### `GeoNode.ciSecureFileRegistries`
+
+Find Ci Secure File registries on this Geo node Available only when feature flag `geo_ci_secure_file_replication` is enabled. This flag is disabled by default, because the feature is experimental and is subject to change without notice.
+
+Returns [`CiSecureFileRegistryConnection`](#cisecurefileregistryconnection).
+
+This field returns a [connection](#connections). It accepts the
+four standard [pagination arguments](#connection-pagination-arguments):
+`before: String`, `after: String`, `first: Int`, `last: Int`.
+
+###### Arguments
+
+| Name | Type | Description |
+| ---- | ---- | ----------- |
+| `ids` | [`[ID!]`](#id) | Filters registries by their ID. |
+
##### `GeoNode.groupWikiRepositoryRegistries`
Find group wiki repository registries on this Geo node.
diff --git a/ee/app/finders/geo/ci_secure_file_registry_finder.rb b/ee/app/finders/geo/ci_secure_file_registry_finder.rb
new file mode 100644
index 0000000000000000000000000000000000000000..ef4f69a9425946cdc2ded77d8276d237647ba4d9
--- /dev/null
+++ b/ee/app/finders/geo/ci_secure_file_registry_finder.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+module Geo
+ class CiSecureFileRegistryFinder
+ include FrameworkRegistryFinder
+ end
+end
diff --git a/ee/app/graphql/resolvers/geo/ci_secure_file_registries_resolver.rb b/ee/app/graphql/resolvers/geo/ci_secure_file_registries_resolver.rb
new file mode 100644
index 0000000000000000000000000000000000000000..2818b4be0efaaff5d1b5f1cb319db8cfdbc93b95
--- /dev/null
+++ b/ee/app/graphql/resolvers/geo/ci_secure_file_registries_resolver.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Resolvers
+ module Geo
+ class CiSecureFileRegistriesResolver < BaseResolver
+ type ::Types::Geo::GeoNodeType.connection_type, null: true
+
+ include RegistriesResolver
+ end
+ end
+end
diff --git a/ee/app/graphql/types/geo/ci_secure_file_registry_type.rb b/ee/app/graphql/types/geo/ci_secure_file_registry_type.rb
new file mode 100644
index 0000000000000000000000000000000000000000..4aae738baa6f431d9421f31cd90d80f792be7423
--- /dev/null
+++ b/ee/app/graphql/types/geo/ci_secure_file_registry_type.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Types
+ module Geo
+ # rubocop:disable Graphql/AuthorizeTypes because it is included
+ class CiSecureFileRegistryType < BaseObject
+ graphql_name 'CiSecureFileRegistry'
+ description 'Represents the Geo replication and verification state of a ci_secure_file.'
+
+ include ::Types::Geo::RegistryType
+
+ field :ci_secure_file_id, GraphQL::Types::ID, null: false, description: 'ID of the Ci Secure File.'
+ end
+ end
+end
diff --git a/ee/app/graphql/types/geo/geo_node_type.rb b/ee/app/graphql/types/geo/geo_node_type.rb
index 314ed1b2b5e003620d9287b7f673a0ebbbcd314d..a0be29b5bfe504c3e87f1cbf1c4508bc2fa6bcff 100644
--- a/ee/app/graphql/types/geo/geo_node_type.rb
+++ b/ee/app/graphql/types/geo/geo_node_type.rb
@@ -7,6 +7,11 @@ class GeoNodeType < BaseObject
authorize :read_geo_node
+ field :ci_secure_file_registries, ::Types::Geo::CiSecureFileRegistryType.connection_type,
+ null: true,
+ resolver: ::Resolvers::Geo::CiSecureFileRegistriesResolver,
+ description: 'Find Ci Secure File registries on this Geo node',
+ feature_flag: :geo_ci_secure_file_replication
field :container_repositories_max_capacity, GraphQL::Types::Int, null: true, description: 'Maximum concurrency of container repository sync for this secondary node.'
field :enabled, GraphQL::Types::Boolean, null: true, description: 'Indicates whether this Geo node is enabled.'
field :files_max_capacity, GraphQL::Types::Int, null: true, description: 'Maximum concurrency of LFS/attachment backfill for this secondary node.'
diff --git a/ee/app/models/concerns/ee/ci/artifactable.rb b/ee/app/models/concerns/ee/ci/artifactable.rb
index 193649891033e8c82d8cc71a56694e974b3af759..e87d66ed713cae0542d840f0ba6c9216a3028ecf 100644
--- a/ee/app/models/concerns/ee/ci/artifactable.rb
+++ b/ee/app/models/concerns/ee/ci/artifactable.rb
@@ -6,8 +6,8 @@ module Artifactable
extend ActiveSupport::Concern
class_methods do
- # @param primary_key_in [Range, Ci::{Pipeline|Job}Artifact] arg to pass to primary_key_in scope
- # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key
+ # @param primary_key_in [Range, Ci::{PipelineArtifact|JobArtifact|SecureFile}] arg to pass to primary_key_in scope
+ # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key
def replicables_for_current_secondary(primary_key_in)
node = ::Gitlab::Geo.current_node
@@ -18,7 +18,7 @@ def replicables_for_current_secondary(primary_key_in)
selective_sync_scope(node, replicables)
end
- # @return [ActiveRecord::Relation] observing object storage settings of the given node
+ # @return [ActiveRecord::Relation] observing object storage settings of the given node
def object_storage_scope(node)
return all if node.sync_object_storage?
@@ -28,7 +28,7 @@ def object_storage_scope(node)
# The primary_key_in in replicables_for_current_secondary method is at most a range of IDs with a maximum of 10_000 records
# between them. We can additionally reduce the batch size to 1_000 just for pipeline artifacts and job artifacts if needed.
#
- # @return [ActiveRecord::Relation] observing selective sync settings of the given node
+ # @return [ActiveRecord::Relation] observing selective sync settings of the given node
def selective_sync_scope(node, replicables)
return replicables unless node.selective_sync?
diff --git a/ee/app/models/ee/ci/secure_file.rb b/ee/app/models/ee/ci/secure_file.rb
new file mode 100644
index 0000000000000000000000000000000000000000..d2bb21e94600f66be02fb3a5c5a06c79ac5cb4a5
--- /dev/null
+++ b/ee/app/models/ee/ci/secure_file.rb
@@ -0,0 +1,66 @@
+# frozen_string_literal: true
+
+module EE
+ # CI::SecureFile EE mixin
+ #
+ # This module is intended to encapsulate EE-specific model logic
+ # and be prepended in the `Ci::SecureFile` model
+ module Ci
+ module SecureFile
+ extend ActiveSupport::Concern
+
+ prepended do
+ include ::Geo::ReplicableModel
+ include ::Geo::VerifiableModel
+ include Artifactable
+
+ delegate(*::Geo::VerificationState::VERIFICATION_METHODS, to: :ci_secure_file_state)
+
+ with_replicator Geo::CiSecureFileReplicator
+
+ has_one :ci_secure_file_state, autosave: false, inverse_of: :ci_secure_file,
+ class_name: 'Geo::CiSecureFileState', foreign_key: :ci_secure_file_id
+
+ after_save :save_verification_details
+
+ scope :with_verification_state, ->(state) {
+ joins(:ci_secure_file_state).where(
+ ci_secure_file_states: {
+ verification_state: verification_state_value(state)
+ }
+ )
+ }
+ scope :checksummed, -> {
+ joins(:ci_secure_file_state).where.not(
+ ci_secure_file_states: { verification_checksum: nil }
+ )
+ }
+ scope :not_checksummed, -> {
+ joins(:ci_secure_file_state).where(
+ ci_secure_file_states: { verification_checksum: nil }
+ )
+ }
+
+ scope :available_verifiables, -> { joins(:ci_secure_file_state) }
+ scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) }
+
+ def verification_state_object
+ ci_secure_file_state
+ end
+ end
+
+ class_methods do
+ extend ::Gitlab::Utils::Override
+
+ override :verification_state_table_class
+ def verification_state_table_class
+ ::Geo::CiSecureFileState
+ end
+ end
+
+ def ci_secure_file_state
+ super || build_ci_secure_file_state
+ end
+ end
+ end
+end
diff --git a/ee/app/models/geo/ci_secure_file_registry.rb b/ee/app/models/geo/ci_secure_file_registry.rb
new file mode 100644
index 0000000000000000000000000000000000000000..6d4e4255ed9768a8191d11b150015770210b15d8
--- /dev/null
+++ b/ee/app/models/geo/ci_secure_file_registry.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Geo
+ class CiSecureFileRegistry < Geo::BaseRegistry
+ include ::Geo::ReplicableRegistry
+ include ::Geo::VerifiableRegistry
+
+ MODEL_CLASS = ::Ci::SecureFile
+ MODEL_FOREIGN_KEY = :ci_secure_file_id
+
+ belongs_to :ci_secure_file, class_name: 'Ci::SecureFile'
+ end
+end
diff --git a/ee/app/models/geo/ci_secure_file_state.rb b/ee/app/models/geo/ci_secure_file_state.rb
new file mode 100644
index 0000000000000000000000000000000000000000..247fa5d084374c5ded107522930fa51d6383cfc5
--- /dev/null
+++ b/ee/app/models/geo/ci_secure_file_state.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+module Geo
+ class CiSecureFileState < Ci::ApplicationRecord
+ include EachBatch
+ include ::Geo::VerificationStateDefinition
+
+ self.primary_key = :ci_secure_file_id
+ self.table_name = :ci_secure_file_states
+
+ belongs_to :ci_secure_file, inverse_of: :ci_secure_file_state, class_name: 'Ci::SecureFile'
+
+ validates :verification_failure, length: { maximum: 255 }
+ validates :verification_state, :ci_secure_file, presence: true
+ end
+end
diff --git a/ee/app/replicators/geo/ci_secure_file_replicator.rb b/ee/app/replicators/geo/ci_secure_file_replicator.rb
new file mode 100644
index 0000000000000000000000000000000000000000..b13e22f0e8aae6ee8b1b1aba1a19e6a7386d5f95
--- /dev/null
+++ b/ee/app/replicators/geo/ci_secure_file_replicator.rb
@@ -0,0 +1,25 @@
+# frozen_string_literal: true
+
+module Geo
+ class CiSecureFileReplicator < Gitlab::Geo::Replicator
+ include ::Geo::BlobReplicatorStrategy
+ extend ::Gitlab::Utils::Override
+
+ def self.model
+ ::Ci::SecureFile
+ end
+
+ def carrierwave_uploader
+ model_record.file
+ end
+
+ override :verification_feature_flag_enabled?
+ def self.verification_feature_flag_enabled?
+ # We are adding verification at the same time as replication, so we
+ # don't need to toggle verification separately from replication. When
+ # the replication feature flag is off, then verification is also off
+ # (see `VerifiableReplicator.verification_enabled?`)
+ true
+ end
+ end
+end
diff --git a/ee/app/workers/geo/secondary/registry_consistency_worker.rb b/ee/app/workers/geo/secondary/registry_consistency_worker.rb
index 10c908caa2c762666d584bda56e9e96b52849f1d..3eebaf2f74c2528e7775e4c3926ea06a1932a05a 100644
--- a/ee/app/workers/geo/secondary/registry_consistency_worker.rb
+++ b/ee/app/workers/geo/secondary/registry_consistency_worker.rb
@@ -31,7 +31,8 @@ class RegistryConsistencyWorker
Geo::UploadRegistry,
Geo::SnippetRepositoryRegistry,
Geo::GroupWikiRepositoryRegistry,
- Geo::PagesDeploymentRegistry
+ Geo::PagesDeploymentRegistry,
+ Geo::CiSecureFileRegistry
].freeze
BATCH_SIZE = 10000
diff --git a/ee/config/feature_flags/development/geo_ci_secure_file_replication.yml b/ee/config/feature_flags/development/geo_ci_secure_file_replication.yml
new file mode 100644
index 0000000000000000000000000000000000000000..db3b2ba9dd722b04368f524b072715cd7d70ee7b
--- /dev/null
+++ b/ee/config/feature_flags/development/geo_ci_secure_file_replication.yml
@@ -0,0 +1,8 @@
+---
+name: geo_ci_secure_file_replication
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/91430
+rollout_issue_url:
+milestone: '15.2'
+type: development
+group: group::incubation
+default_enabled: false
diff --git a/ee/lib/gitlab/geo.rb b/ee/lib/gitlab/geo.rb
index eb0485d17fa41ef6fe716bdc8aa50c5016b75cb5..7de3591fbbf1cafe07f975dc328fea64dc593c1b 100644
--- a/ee/lib/gitlab/geo.rb
+++ b/ee/lib/gitlab/geo.rb
@@ -32,7 +32,8 @@ module Geo
::Geo::PipelineArtifactReplicator,
::Geo::PagesDeploymentReplicator,
::Geo::UploadReplicator,
- ::Geo::JobArtifactReplicator
+ ::Geo::JobArtifactReplicator,
+ ::Geo::CiSecureFileReplicator
].freeze
# We "regenerate" an 1hour valid JWT every 30 minutes, resulting in
diff --git a/ee/spec/factories/ci/secure_files.rb b/ee/spec/factories/ci/secure_files.rb
new file mode 100644
index 0000000000000000000000000000000000000000..629350e35470567a743e3e123856468ebe5e134b
--- /dev/null
+++ b/ee/spec/factories/ci/secure_files.rb
@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+ factory :ee_ci_secure_file, class: '::Ci::SecureFile', parent: :ci_secure_file do
+ trait(:verification_succeeded) do
+ with_file
+ verification_checksum { 'abc' }
+ verification_state { Ci::SecureFile.verification_state_value(:verification_succeeded) }
+ end
+
+ trait(:verification_failed) do
+ with_file
+ verification_failure { 'Could not calculate the checksum' }
+ verification_state { Ci::SecureFile.verification_state_value(:verification_failed) }
+ end
+ end
+end
diff --git a/ee/spec/factories/geo/ci_secure_file_registry.rb b/ee/spec/factories/geo/ci_secure_file_registry.rb
new file mode 100644
index 0000000000000000000000000000000000000000..a17445b7f06cb27e8c7177df8bd7eb166ab784fd
--- /dev/null
+++ b/ee/spec/factories/geo/ci_secure_file_registry.rb
@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+ factory :geo_ci_secure_file_registry, class: 'Geo::CiSecureFileRegistry' do
+ ci_secure_file # This association should have data, like a file or repository
+ state { Geo::CiSecureFileRegistry.state_value(:pending) }
+
+ trait :synced do
+ state { Geo::CiSecureFileRegistry.state_value(:synced) }
+ last_synced_at { 5.days.ago }
+ end
+
+ trait :failed do
+ state { Geo::CiSecureFileRegistry.state_value(:failed) }
+ last_synced_at { 1.day.ago }
+ retry_count { 2 }
+ last_sync_failure { 'Random error' }
+ end
+
+ trait :started do
+ state { Geo::CiSecureFileRegistry.state_value(:started) }
+ last_synced_at { 1.day.ago }
+ retry_count { 0 }
+ end
+
+ trait :verification_succeeded do
+ verification_checksum { 'e079a831cab27bcda7d81cd9b48296d0c3dd92ef' }
+ verification_state { Geo::CiSecureFileRegistry.verification_state_value(:verification_succeeded) }
+ verified_at { 5.days.ago }
+ end
+ end
+end
diff --git a/ee/spec/factories/geo/ci_secure_file_states.rb b/ee/spec/factories/geo/ci_secure_file_states.rb
new file mode 100644
index 0000000000000000000000000000000000000000..a390bd837ec9b921a98b6fab00957ad2148c9520
--- /dev/null
+++ b/ee/spec/factories/geo/ci_secure_file_states.rb
@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+FactoryBot.define do
+ factory :geo_ci_secure_file_state, class: 'Geo::CiSecureFileState' do
+ ci_secure_file
+
+ trait(:checksummed) do
+ verification_checksum { 'abc' }
+ end
+
+ trait(:checksum_failure) do
+ verification_failure { 'Could not calculate the checksum' }
+ end
+ end
+end
diff --git a/ee/spec/finders/geo/ci_secure_file_registry_finder_spec.rb b/ee/spec/finders/geo/ci_secure_file_registry_finder_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..764c29c5d591338c42aed4b1969025035654454a
--- /dev/null
+++ b/ee/spec/finders/geo/ci_secure_file_registry_finder_spec.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::CiSecureFileRegistryFinder do
+ it_behaves_like 'a framework registry finder', :geo_ci_secure_file_registry
+end
diff --git a/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json b/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json
index 50284970513a1ee516fa76168f3666c1128b5bc6..213c4311da0743d73cc3d2d4fc5abdff8d19eeb1 100644
--- a/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json
+++ b/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json
@@ -29,6 +29,18 @@
"job_artifacts_verified_count",
"job_artifacts_verification_failed_count",
"job_artifacts_verified_in_percentage",
+ "ci_secure_files_count",
+ "ci_secure_files_checksum_total_count",
+ "ci_secure_files_checksummed_count",
+ "ci_secure_files_checksum_failed_count",
+ "ci_secure_files_synced_count",
+ "ci_secure_files_failed_count",
+ "ci_secure_files_registry_count",
+ "ci_secure_files_verification_total_count",
+ "ci_secure_files_verified_count",
+ "ci_secure_files_verification_failed_count",
+ "ci_secure_files_synced_in_percentage",
+ "ci_secure_files_verified_in_percentage",
"db_replication_lag_seconds",
"container_repositories_replication_enabled",
"container_repositories_count",
@@ -219,6 +231,18 @@
"job_artifacts_verified_count": { "type": ["integer", "null"] },
"job_artifacts_verification_failed_count": { "type": ["integer", "null"] },
"job_artifacts_verified_in_percentage": { "type": "string" },
+ "ci_secure_files_count": { "type": "integer" },
+ "ci_secure_files_failed_count": { "type": ["integer", "null"] },
+ "ci_secure_files_synced_count": { "type": ["integer", "null"] },
+ "ci_secure_files_synced_in_percentage": { "type": "string" },
+ "ci_secure_files_checksum_total_count": { "type": ["integer", "null"] },
+ "ci_secure_files_checksummed_count": { "type": ["integer", "null"] },
+ "ci_secure_files_checksum_failed_count": { "type": ["integer", "null"] },
+ "ci_secure_files_registry_count": { "type": ["integer", "null"] },
+ "ci_secure_files_verification_total_count": { "type": ["integer", "null"] },
+ "ci_secure_files_verified_count": { "type": ["integer", "null"] },
+ "ci_secure_files_verification_failed_count": { "type": ["integer", "null"] },
+ "ci_secure_files_verified_in_percentage": { "type": "string" },
"container_repositories_replication_enabled": { "type": ["boolean", "null"] },
"container_repositories_count": { "type": "integer" },
"container_repositories_failed_count": { "type": ["integer", "null"] },
diff --git a/ee/spec/graphql/resolvers/geo/ci_secure_file_registries_resolver_spec.rb b/ee/spec/graphql/resolvers/geo/ci_secure_file_registries_resolver_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..2a7c26a68d052ec4eafb9d075d1836e57b47803a
--- /dev/null
+++ b/ee/spec/graphql/resolvers/geo/ci_secure_file_registries_resolver_spec.rb
@@ -0,0 +1,7 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Resolvers::Geo::CiSecureFileRegistriesResolver do
+ it_behaves_like 'a Geo registries resolver', :geo_ci_secure_file_registry
+end
diff --git a/ee/spec/graphql/types/geo/ci_secure_file_registry_type_spec.rb b/ee/spec/graphql/types/geo/ci_secure_file_registry_type_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..2c80c7463fd29b7deb80700311ad8a24a864ac06
--- /dev/null
+++ b/ee/spec/graphql/types/geo/ci_secure_file_registry_type_spec.rb
@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe GitlabSchema.types['CiSecureFileRegistry'] do
+ it_behaves_like 'a Geo registry type'
+
+ it 'has the expected fields (other than those included in RegistryType)' do
+ expected_fields = %i[ci_secure_file_id]
+
+ expect(described_class).to have_graphql_fields(*expected_fields).at_least
+ end
+end
diff --git a/ee/spec/graphql/types/geo/geo_node_type_spec.rb b/ee/spec/graphql/types/geo/geo_node_type_spec.rb
index 6d9becaa901c083d8bc4da165fb80c6c2bfabcca..676251819e0cb5f9efaabb06cd41db3d0076068e 100644
--- a/ee/spec/graphql/types/geo/geo_node_type_spec.rb
+++ b/ee/spec/graphql/types/geo/geo_node_type_spec.rb
@@ -15,7 +15,7 @@
package_file_registries snippet_repository_registries
terraform_state_version_registries group_wiki_repository_registries
pages_deployment_registries lfs_object_registries pipeline_artifact_registries
- upload_registries job_artifact_registries
+ upload_registries job_artifact_registries ci_secure_file_registries
]
expect(described_class).to have_graphql_fields(*expected_fields)
diff --git a/ee/spec/models/ee/ci/secure_file_spec.rb b/ee/spec/models/ee/ci/secure_file_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..5f11c315311136ddc460bf528f32057b39df2f5d
--- /dev/null
+++ b/ee/spec/models/ee/ci/secure_file_spec.rb
@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Ci::SecureFile do
+ using RSpec::Parameterized::TableSyntax
+ include EE::GeoHelpers
+
+ include_examples 'a replicable model with a separate table for verification state' do
+ let(:project) { create(:project) }
+ let(:verifiable_model_record) { build(:ci_secure_file, project: project) }
+
+ let(:unverifiable_model_record) do
+ stub_ci_secure_file_object_storage
+ file = build(:ci_secure_file, :remote_store, project: project)
+ stub_ci_secure_file_object_storage(enabled: false)
+
+ file
+ end
+ end
+
+ describe '#replicables_for_current_secondary' do
+ # Selective sync is configured relative to the secure file's project.
+ #
+ # Permutations of sync_object_storage combined with object-stored-artifacts
+ # are tested in code, because the logic is simple, and to do it in the table
+ # would quadruple its size and have too much duplication.
+ where(:selective_sync_namespaces, :selective_sync_shards, :factory, :project_factory, :include_expectation) do
+ nil | nil | [:ci_secure_file] | [:project] | true
+ # selective sync by shard
+ nil | :model | [:ci_secure_file] | [:project] | true
+ nil | :other | [:ci_secure_file] | [:project] | false
+ # selective sync by namespace
+ :model_parent | nil | [:ci_secure_file] | [:project] | true
+ :model_parent_parent | nil | [:ci_secure_file] | [:project, :in_subgroup] | true
+ :other | nil | [:ci_secure_file] | [:project] | false
+ :other | nil | [:ci_secure_file] | [:project, :in_subgroup] | false
+ end
+
+ with_them do
+ subject(:ci_secure_file_included) { described_class.replicables_for_current_secondary(ci_secure_file).exists? }
+
+ let(:project) { create(*project_factory) } # rubocop:disable Rails/SaveBang
+ let(:node) do
+ create(:geo_node_with_selective_sync_for,
+ model: project,
+ namespaces: selective_sync_namespaces,
+ shards: selective_sync_shards,
+ sync_object_storage: sync_object_storage)
+ end
+
+ before do
+ stub_current_geo_node(node)
+ end
+
+ context 'when sync object storage is enabled' do
+ let(:sync_object_storage) { true }
+
+ context 'when the ci secure file is locally stored' do
+ let(:ci_secure_file) { create(*factory, project: project) }
+
+ it { is_expected.to eq(include_expectation) }
+ end
+
+ context 'when the ci secure file is object stored' do
+ let(:ci_secure_file) { create(*factory, :remote_store, project: project) }
+
+ it { is_expected.to eq(include_expectation) }
+ end
+ end
+
+ context 'when sync object storage is disabled' do
+ let(:sync_object_storage) { false }
+
+ context 'when the ci secure file is locally stored' do
+ let(:ci_secure_file) { create(*factory, project: project) }
+
+ it { is_expected.to eq(include_expectation) }
+ end
+
+ context 'when the ci secure file is object stored' do
+ let(:ci_secure_file) { create(*factory, :remote_store, project: project) }
+
+ it { is_expected.to be_falsey }
+ end
+ end
+ end
+ end
+end
diff --git a/ee/spec/models/geo/ci_secure_file_registry_spec.rb b/ee/spec/models/geo/ci_secure_file_registry_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..1c31948f25add9d1b3a531d8f4f9a81fda9cc668
--- /dev/null
+++ b/ee/spec/models/geo/ci_secure_file_registry_spec.rb
@@ -0,0 +1,14 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::CiSecureFileRegistry, :geo, type: :model do
+ let_it_be(:registry) { create(:geo_ci_secure_file_registry) }
+
+ specify 'factory is valid' do
+ expect(registry).to be_valid
+ end
+
+ include_examples 'a Geo framework registry'
+ include_examples 'a Geo verifiable registry'
+end
diff --git a/ee/spec/models/geo_node_status_spec.rb b/ee/spec/models/geo_node_status_spec.rb
index ae7b641c0c8b42339ada359f8bd6be62e9c6b54c..22cab9fb08a70bf88751f7f36ba6a5eb43d44769 100644
--- a/ee/spec/models/geo_node_status_spec.rb
+++ b/ee/spec/models/geo_node_status_spec.rb
@@ -1031,6 +1031,7 @@
Geo::PagesDeploymentReplicator | :pages_deployment | :geo_pages_deployment_registry
Geo::UploadReplicator | :upload | :geo_upload_registry
Geo::JobArtifactReplicator | :ci_job_artifact | :geo_job_artifact_registry
+ Geo::CiSecureFileReplicator | :ci_secure_file | :geo_ci_secure_file_registry
end
with_them do
diff --git a/ee/spec/replicators/geo/ci_secure_file_replicator_spec.rb b/ee/spec/replicators/geo/ci_secure_file_replicator_spec.rb
new file mode 100644
index 0000000000000000000000000000000000000000..15fcefdfe2ff8c4fa1eda9b646fd3aa2acf423ac
--- /dev/null
+++ b/ee/spec/replicators/geo/ci_secure_file_replicator_spec.rb
@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::CiSecureFileReplicator do
+ let(:project) { create(:project) }
+ let(:model_record) { create(:ci_secure_file, project: project) }
+
+ include_examples 'a blob replicator'
+ include_examples 'a verifiable replicator'
+end
diff --git a/ee/spec/requests/api/graphql/geo/registries_spec.rb b/ee/spec/requests/api/graphql/geo/registries_spec.rb
index 7c4cdd0680e6d5dfe54931b2bf27274f2be7b234..7c00649dedf0dacea5a83eb4a46a0c3b75434dbf 100644
--- a/ee/spec/requests/api/graphql/geo/registries_spec.rb
+++ b/ee/spec/requests/api/graphql/geo/registries_spec.rb
@@ -65,4 +65,11 @@
registry_factory: :geo_job_artifact_registry,
registry_foreign_key_field_name: 'artifactId'
}
+
+ it_behaves_like 'gets registries for', {
+ field_name: 'ciSecureFileRegistries',
+ registry_class_name: 'CiSecureFileRegistry',
+ registry_factory: :geo_ci_secure_file_registry,
+ registry_foreign_key_field_name: 'ciSecureFileId'
+ }
end
diff --git a/ee/spec/services/geo/registry_consistency_service_spec.rb b/ee/spec/services/geo/registry_consistency_service_spec.rb
index d182ee2bb1d625f16ea94c291eb980b48734a8c2..187ed26946c572bee6c154f9129bc52a9970377d 100644
--- a/ee/spec/services/geo/registry_consistency_service_spec.rb
+++ b/ee/spec/services/geo/registry_consistency_service_spec.rb
@@ -21,7 +21,8 @@ def model_class_factory_name(registry_class)
Geo::MergeRequestDiffRegistry => :external_merge_request_diff,
Geo::PackageFileRegistry => :package_file,
Geo::UploadRegistry => :upload,
- Geo::JobArtifactRegistry => :ci_job_artifact
+ Geo::JobArtifactRegistry => :ci_job_artifact,
+ Geo::CiSecureFileRegistry => :ci_secure_file
}.fetch(registry_class, default_factory_name)
end
diff --git a/ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb b/ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb
index 94db718fda30c343bfaa1e4d9963e27e887b01c4..22ca629676c0856bbed29b8ab90682d328155421 100644
--- a/ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb
+++ b/ee/spec/workers/geo/secondary/registry_consistency_worker_spec.rb
@@ -88,6 +88,7 @@
pipeline_artifact = create(:ci_pipeline_artifact)
upload = create(:upload)
pages_deployment = create(:pages_deployment)
+ ci_secure_file = create(:ci_secure_file)
expect(Geo::ContainerRepositoryRegistry.where(container_repository_id: container_repository.id).count).to eq(0)
expect(Geo::DesignRegistry.where(project_id: project.id).count).to eq(0)
@@ -101,6 +102,7 @@
expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(0)
expect(Geo::PagesDeploymentRegistry.where(pages_deployment: pages_deployment.id).count).to eq(0)
expect(Geo::JobArtifactRegistry.where(job_artifact: job_artifact.id).count).to eq(0)
+ expect(Geo::CiSecureFileRegistry.where(ci_secure_file: ci_secure_file.id).count).to eq(0)
subject.perform
@@ -116,6 +118,7 @@
expect(Geo::UploadRegistry.where(file_id: upload.id).count).to eq(1)
expect(Geo::PagesDeploymentRegistry.where(pages_deployment: pages_deployment.id).count).to eq(1)
expect(Geo::JobArtifactRegistry.where(job_artifact: job_artifact.id).count).to eq(1)
+ expect(Geo::CiSecureFileRegistry.where(ci_secure_file: ci_secure_file.id).count).to eq(1)
end
context 'when the current Geo node is disabled or primary' do
diff --git a/spec/factories/ci/secure_files.rb b/spec/factories/ci/secure_files.rb
index 9afec5db85879aec839cdac0b74d5b6c9dec2b6c..74988202c71477a17e54ac79b1cbfec843dd4d0a 100644
--- a/spec/factories/ci/secure_files.rb
+++ b/spec/factories/ci/secure_files.rb
@@ -6,5 +6,11 @@
file { fixture_file_upload('spec/fixtures/ci_secure_files/upload-keystore.jks', 'application/octet-stream') }
checksum { 'foo1234' }
project
+
+ trait :remote_store do
+ after(:create) do |ci_secure_file|
+ ci_secure_file.update!(file_store: ObjectStorage::Store::REMOTE)
+ end
+ end
end
end