diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index d5e88f2be5bea10d06e939f60041d49b860d9871..705e377f66547f3e9647644ef590e33b4a496393 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -131,7 +131,6 @@ class JobArtifact < ApplicationRecord update_project_statistics project_statistics_name: :build_artifacts_size scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } - scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) } scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } diff --git a/app/models/ci/pipeline_artifact.rb b/app/models/ci/pipeline_artifact.rb index f538a4cd8089547fb556a1d28bd6c89a0ea182f5..17c47318146466d94bb83f60b12337fbaf81f4e9 100644 --- a/app/models/ci/pipeline_artifact.rb +++ b/app/models/ci/pipeline_artifact.rb @@ -57,3 +57,5 @@ def present end end end + +Ci::PipelineArtifact.prepend_if_ee('::EE::Ci::PipelineArtifact') diff --git a/app/models/concerns/ci/artifactable.rb b/app/models/concerns/ci/artifactable.rb index cbe7d3b6abb1e848b1b698e1276be7c9eff5f165..e7e733d47ec04dbdca1e66c2ed3801c16534f928 100644 --- a/app/models/concerns/ci/artifactable.rb +++ b/app/models/concerns/ci/artifactable.rb @@ -20,6 +20,8 @@ module Artifactable scope :expired_before, -> (timestamp) { where(arel_table[:expire_at].lt(timestamp)) } scope :expired, -> (limit) { expired_before(Time.current).limit(limit) } + scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) } + scope :project_id_in, ->(ids) { where(project_id: ids) } end def each_blob(&blk) diff --git a/changelogs/unreleased/mo-pipeline-artifact-geo-poc.yml b/changelogs/unreleased/mo-pipeline-artifact-geo-poc.yml new file mode 100644 index 0000000000000000000000000000000000000000..97661f99482fee24e1e40fdef6100cbeb3e8f277 --- /dev/null +++ b/changelogs/unreleased/mo-pipeline-artifact-geo-poc.yml @@ -0,0 +1,5 @@ +--- +title: Include pipeline artifacts in Geo replication +merge_request: 56423 +author: +type: added diff --git a/config/feature_flags/development/geo_pipeline_artifact_replication.yml b/config/feature_flags/development/geo_pipeline_artifact_replication.yml new file mode 100644 index 0000000000000000000000000000000000000000..c8aa0e3effb653d02e594589bbbf086ab9974416 --- /dev/null +++ b/config/feature_flags/development/geo_pipeline_artifact_replication.yml @@ -0,0 +1,8 @@ +--- +name: geo_pipeline_artifact_replication +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/238464 +rollout_issue_url: +milestone: '13.10' +type: development +group: group::geo +default_enabled: false diff --git a/config/initializers_before_autoloader/000_inflections.rb b/config/initializers_before_autoloader/000_inflections.rb index 308b38c1ba249db11ed569d4a1481428f2fe63c4..93cf9e01b96f381608afdee39b150eec5a425d3c 100644 --- a/config/initializers_before_autoloader/000_inflections.rb +++ b/config/initializers_before_autoloader/000_inflections.rb @@ -24,6 +24,7 @@ job_artifact_registry lfs_object_registry package_file_registry + pipeline_artifact_registry project_auto_devops project_registry project_statistics diff --git a/db/migrate/20210323202729_add_verification_state_to_pipeline_artifact.rb b/db/migrate/20210323202729_add_verification_state_to_pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..74bbd8ffedbb07074f28244081c3655a62468d20 --- /dev/null +++ b/db/migrate/20210323202729_add_verification_state_to_pipeline_artifact.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddVerificationStateToPipelineArtifact < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + change_table(:ci_pipeline_artifacts) do |t| + t.integer :verification_state, default: 0, limit: 2, null: false + t.integer :verification_retry_count, limit: 2 + t.column :verification_started_at, :datetime_with_timezone + t.column :verification_retry_at, :datetime_with_timezone + t.column :verified_at, :datetime_with_timezone + t.binary :verification_checksum, using: 'verification_checksum::bytea' + + t.text :verification_failure # rubocop:disable Migration/AddLimitToTextColumns + end + end +end diff --git a/db/migrate/20210323203055_add_verification_failure_limit_to_pipeline_artifact.rb b/db/migrate/20210323203055_add_verification_failure_limit_to_pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..77c87743e7e5ffeca8d59c90577f63504b37de33 --- /dev/null +++ b/db/migrate/20210323203055_add_verification_failure_limit_to_pipeline_artifact.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddVerificationFailureLimitToPipelineArtifact < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + CONSTRAINT_NAME = 'ci_pipeline_artifacts_verification_failure_text_limit' + + def up + add_text_limit :ci_pipeline_artifacts, :verification_failure, 255, constraint_name: CONSTRAINT_NAME + end + + def down + remove_check_constraint(:ci_pipeline_artifacts, CONSTRAINT_NAME) + end +end diff --git a/db/migrate/20210323203227_add_verification_indexes_to_pipeline_artifacts.rb b/db/migrate/20210323203227_add_verification_indexes_to_pipeline_artifacts.rb new file mode 100644 index 0000000000000000000000000000000000000000..fbda76c860e096c4875b6804fe865bb1be61f506 --- /dev/null +++ b/db/migrate/20210323203227_add_verification_indexes_to_pipeline_artifacts.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class AddVerificationIndexesToPipelineArtifacts < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + VERIFICATION_STATE_INDEX_NAME = "index_ci_pipeline_artifacts_verification_state" + PENDING_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_pending_verification" + FAILED_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_needs_verification" + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_pipeline_artifacts, :verification_state, name: VERIFICATION_STATE_INDEX_NAME + add_concurrent_index :ci_pipeline_artifacts, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME + add_concurrent_index :ci_pipeline_artifacts, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME + add_concurrent_index :ci_pipeline_artifacts, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_pipeline_artifacts, VERIFICATION_STATE_INDEX_NAME + remove_concurrent_index_by_name :ci_pipeline_artifacts, PENDING_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :ci_pipeline_artifacts, FAILED_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :ci_pipeline_artifacts, NEEDS_VERIFICATION_INDEX_NAME + end +end diff --git a/db/schema_migrations/20210323202729 b/db/schema_migrations/20210323202729 new file mode 100644 index 0000000000000000000000000000000000000000..4ceab523949d6391d6bd24dfab47952c5745b224 --- /dev/null +++ b/db/schema_migrations/20210323202729 @@ -0,0 +1 @@ +3866cb66b2ab22f326f8a542538c58a0410cdb65fbefb91f5c15863a171fb4fe \ No newline at end of file diff --git a/db/schema_migrations/20210323203055 b/db/schema_migrations/20210323203055 new file mode 100644 index 0000000000000000000000000000000000000000..0ce0ca10b86797e0e8d0346c46413f60ebe1b83e --- /dev/null +++ b/db/schema_migrations/20210323203055 @@ -0,0 +1 @@ +483153eea0df2bfd850413119ebf818a478afc821e70a6d5cb8eb1bc5bc4683a \ No newline at end of file diff --git a/db/schema_migrations/20210323203227 b/db/schema_migrations/20210323203227 new file mode 100644 index 0000000000000000000000000000000000000000..94a7a245eb18eecb3bfef578fa46bf80ca9cf6a2 --- /dev/null +++ b/db/schema_migrations/20210323203227 @@ -0,0 +1 @@ +b7c3bc4b39ca40234f3c675ba3f2a6100e2d2389b0e85e3eb02ac66bff64582d \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index a51e47a9317c0efa4674a5ccd9afb4e0f0b88436..a74291934484a5bd6e8c904ad243e4cc4724a6ec 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10672,8 +10672,16 @@ CREATE TABLE ci_pipeline_artifacts ( file_format smallint NOT NULL, file text, expire_at timestamp with time zone, + verification_state smallint DEFAULT 0 NOT NULL, + verification_retry_count smallint, + verification_started_at timestamp with time zone, + verification_retry_at timestamp with time zone, + verified_at timestamp with time zone, + verification_checksum bytea, + verification_failure text, CONSTRAINT check_191b5850ec CHECK ((char_length(file) <= 255)), - CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL)) + CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL)), + CONSTRAINT ci_pipeline_artifacts_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255)) ); CREATE SEQUENCE ci_pipeline_artifacts_id_seq @@ -22108,6 +22116,10 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date); +CREATE INDEX index_ci_pipeline_artifacts_failed_verification ON ci_pipeline_artifacts USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3); + +CREATE INDEX index_ci_pipeline_artifacts_needs_verification ON ci_pipeline_artifacts USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3)); + CREATE INDEX index_ci_pipeline_artifacts_on_expire_at ON ci_pipeline_artifacts USING btree (expire_at); CREATE INDEX index_ci_pipeline_artifacts_on_pipeline_id ON ci_pipeline_artifacts USING btree (pipeline_id); @@ -22116,6 +22128,10 @@ CREATE UNIQUE INDEX index_ci_pipeline_artifacts_on_pipeline_id_and_file_type ON CREATE INDEX index_ci_pipeline_artifacts_on_project_id ON ci_pipeline_artifacts USING btree (project_id); +CREATE INDEX index_ci_pipeline_artifacts_pending_verification ON ci_pipeline_artifacts USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0); + +CREATE INDEX index_ci_pipeline_artifacts_verification_state ON ci_pipeline_artifacts USING btree (verification_state); + CREATE INDEX index_ci_pipeline_chat_data_on_chat_name_id ON ci_pipeline_chat_data USING btree (chat_name_id); CREATE UNIQUE INDEX index_ci_pipeline_chat_data_on_pipeline_id ON ci_pipeline_chat_data USING btree (pipeline_id); diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 0401680b016f8cb1724271de1806447dde333ba5..dbd0c2c918193ad645bfc6629c029d37b682a7c9 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -388,7 +388,20 @@ Example response: "group_wiki_repositories_checksum_failed_count": 0, "group_wiki_repositories_registry_count": 10, "group_wiki_repositories_synced_count": 6, - "group_wiki_repositories_failed_count": 3 + "group_wiki_repositories_failed_count": 3, + "pipeline_artifacts_count": 5, + "pipeline_artifacts_checksum_total_count": 5, + "pipeline_artifacts_checksummed_count": 5, + "pipeline_artifacts_checksum_failed_count": 0, + "pipeline_artifacts_synced_count": null, + "pipeline_artifacts_failed_count": null, + "pipeline_artifacts_registry_count": null, + "pipeline_artifacts_verification_total_count": null, + "pipeline_artifacts_verified_count": null, + "pipeline_artifacts_verification_failed_count": null, + "pipeline_artifacts_synced_in_percentage": "0.00%", + "pipeline_artifacts_verified_in_percentage": "0.00%", + }, { "geo_node_id": 2, @@ -487,7 +500,19 @@ Example response: "group_wiki_repositories_checksum_failed_count": 0, "group_wiki_repositories_registry_count": 10, "group_wiki_repositories_synced_count": 6, - "group_wiki_repositories_failed_count": 3 + "group_wiki_repositories_failed_count": 3, + "pipeline_artifacts_count": 5, + "pipeline_artifacts_checksum_total_count": 5, + "pipeline_artifacts_checksummed_count": 5, + "pipeline_artifacts_checksum_failed_count": 0, + "pipeline_artifacts_synced_count": null, + "pipeline_artifacts_failed_count": null, + "pipeline_artifacts_registry_count": null, + "pipeline_artifacts_verification_total_count": null, + "pipeline_artifacts_verified_count": null, + "pipeline_artifacts_verification_failed_count": null, + "pipeline_artifacts_synced_in_percentage": "0.00%", + "pipeline_artifacts_verified_in_percentage": "0.00%", } ] ``` diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 331d3be2b8c3c9bff79ff35b112a6122237a346c..95f003d7db4daa2fa45cb74df7ed62704b4cdb23 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -3016,6 +3016,7 @@ Represents an external issue. | `minimumReverificationInterval` | [`Int`](#int) | The interval (in days) in which the repository verification is valid. Once expired, it will be reverified. | | `name` | [`String`](#string) | The unique identifier for this Geo node. | | `packageFileRegistries` | [`PackageFileRegistryConnection`](#packagefileregistryconnection) | Package file registries of the GeoNode. | +| `pipelineArtifactRegistries` | [`PipelineArtifactRegistryConnection`](#pipelineartifactregistryconnection) | Find pipeline artifact registries on this Geo node. Available only when feature flag `geo_pipeline_artifact_replication` is enabled. | | `primary` | [`Boolean`](#boolean) | Indicates whether this Geo node is the primary. | | `reposMaxCapacity` | [`Int`](#int) | The maximum concurrency of repository backfill for this secondary node. | | `selectiveSyncNamespaces` | [`NamespaceConnection`](#namespaceconnection) | The namespaces that should be synced, if `selective_sync_type` == `namespaces`. | @@ -4624,6 +4625,40 @@ Information about pagination in a connection. | `yearPipelinesSuccessful` | [`[Int!]`](#int) | Total yearly successful pipeline count. | | `yearPipelinesTotals` | [`[Int!]`](#int) | Total yearly pipeline count. | +### `PipelineArtifactRegistry` + +Represents the Geo sync and verification state of a pipeline artifact. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `createdAt` | [`Time`](#time) | Timestamp when the PipelineArtifactRegistry was created | +| `id` | [`ID!`](#id) | ID of the PipelineArtifactRegistry | +| `lastSyncFailure` | [`String`](#string) | Error message during sync of the PipelineArtifactRegistry | +| `lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the PipelineArtifactRegistry | +| `pipelineArtifactId` | [`ID!`](#id) | ID of the pipeline artifact. | +| `retryAt` | [`Time`](#time) | Timestamp after which the PipelineArtifactRegistry should be resynced | +| `retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the PipelineArtifactRegistry | +| `state` | [`RegistryState`](#registrystate) | Sync state of the PipelineArtifactRegistry | + +### `PipelineArtifactRegistryConnection` + +The connection type for PipelineArtifactRegistry. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `edges` | [`[PipelineArtifactRegistryEdge]`](#pipelineartifactregistryedge) | A list of edges. | +| `nodes` | [`[PipelineArtifactRegistry]`](#pipelineartifactregistry) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +### `PipelineArtifactRegistryEdge` + +An edge in a connection. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`PipelineArtifactRegistry`](#pipelineartifactregistry) | The item at the end of the edge. | + ### `PipelineCancelPayload` Autogenerated return type of PipelineCancel. diff --git a/ee/app/finders/geo/pipeline_artifact_registry_finder.rb b/ee/app/finders/geo/pipeline_artifact_registry_finder.rb new file mode 100644 index 0000000000000000000000000000000000000000..ec68f5248be12956f947ba4612f2bda43eaa1457 --- /dev/null +++ b/ee/app/finders/geo/pipeline_artifact_registry_finder.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Geo + class PipelineArtifactRegistryFinder + include FrameworkRegistryFinder + end +end diff --git a/ee/app/graphql/resolvers/geo/pipeline_artifact_registries_resolver.rb b/ee/app/graphql/resolvers/geo/pipeline_artifact_registries_resolver.rb new file mode 100644 index 0000000000000000000000000000000000000000..61c51d75ae91243534714582340adfe491c9c054 --- /dev/null +++ b/ee/app/graphql/resolvers/geo/pipeline_artifact_registries_resolver.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Resolvers + module Geo + class PipelineArtifactRegistriesResolver < BaseResolver + type ::Types::Geo::GeoNodeType.connection_type, null: true + + include RegistriesResolver + end + end +end diff --git a/ee/app/graphql/types/geo/geo_node_type.rb b/ee/app/graphql/types/geo/geo_node_type.rb index eba46fe195aa713e3e841d4c54ab5d867b5787a4..82456ce4651b5ec0577e47734321ddd538a4b844 100644 --- a/ee/app/graphql/types/geo/geo_node_type.rb +++ b/ee/app/graphql/types/geo/geo_node_type.rb @@ -42,6 +42,11 @@ class GeoNodeType < BaseObject null: true, resolver: ::Resolvers::Geo::GroupWikiRepositoryRegistriesResolver, description: 'Find group wiki repository registries on this Geo node.' + field :pipeline_artifact_registries, ::Types::Geo::PipelineArtifactRegistryType.connection_type, + null: true, + resolver: ::Resolvers::Geo::PipelineArtifactRegistriesResolver, + description: 'Find pipeline artifact registries on this Geo node.', + feature_flag: :geo_pipeline_artifact_replication end end end diff --git a/ee/app/graphql/types/geo/pipeline_artifact_registry_type.rb b/ee/app/graphql/types/geo/pipeline_artifact_registry_type.rb new file mode 100644 index 0000000000000000000000000000000000000000..1036d51fb06eba7fca9017ea7ac322588d6673f0 --- /dev/null +++ b/ee/app/graphql/types/geo/pipeline_artifact_registry_type.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Types + module Geo + # rubocop:disable Graphql/AuthorizeTypes because it is included + class PipelineArtifactRegistryType < BaseObject + include ::Types::Geo::RegistryType + + graphql_name 'PipelineArtifactRegistry' + description 'Represents the Geo sync and verification state of a pipeline artifact' + + field :pipeline_artifact_id, GraphQL::ID_TYPE, null: false, description: 'ID of the pipeline artifact.' + end + end +end diff --git a/ee/app/models/ee/ci/job_artifact.rb b/ee/app/models/ee/ci/job_artifact.rb index 2cc0700e69abe1198d5a98602130ae8e58a40de7..f39ba5767e73884965238ac5cd8f3f51626c9e06 100644 --- a/ee/app/models/ee/ci/job_artifact.rb +++ b/ee/app/models/ee/ci/job_artifact.rb @@ -23,7 +23,6 @@ module Ci::JobArtifact API_FUZZING_REPORT_TYPES = %w[api_fuzzing].freeze BROWSER_PERFORMANCE_REPORT_FILE_TYPES = %w[browser_performance performance].freeze - scope :project_id_in, ->(ids) { where(project_id: ids) } scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :security_reports, -> (file_types: SECURITY_REPORT_FILE_TYPES) do diff --git a/ee/app/models/ee/ci/pipeline_artifact.rb b/ee/app/models/ee/ci/pipeline_artifact.rb new file mode 100644 index 0000000000000000000000000000000000000000..b87dbd0fb9121019845a98963de1e0dda21c70dc --- /dev/null +++ b/ee/app/models/ee/ci/pipeline_artifact.rb @@ -0,0 +1,40 @@ +# frozen_string_literal: true + +module EE + module Ci + module PipelineArtifact + extend ActiveSupport::Concern + + prepended do + include ::Gitlab::Geo::ReplicableModel + include ::Gitlab::Geo::VerificationState + + with_replicator ::Geo::PipelineArtifactReplicator + end + + class_methods do + # @param primary_key_in [Range, Ci::PipelineArtifact] arg to pass to primary_key_in scope + # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key + def replicables_for_current_secondary(primary_key_in) + node = ::Gitlab::Geo.current_node + + primary_key_in(primary_key_in) + .merge(selective_sync_scope(node)) + .merge(object_storage_scope(node)) + end + + def object_storage_scope(node) + return all if node.sync_object_storage? + + with_files_stored_locally + end + + def selective_sync_scope(node) + return all unless node.selective_sync? + + project_id_in(node.projects) + end + end + end + end +end diff --git a/ee/app/models/ee/terraform/state_version.rb b/ee/app/models/ee/terraform/state_version.rb index 04572af4edb2142bc7cc58c2561275f0237e287b..22adf92ce9ccc56af3a119d6655d444678601552 100644 --- a/ee/app/models/ee/terraform/state_version.rb +++ b/ee/app/models/ee/terraform/state_version.rb @@ -7,6 +7,7 @@ module StateVersion prepended do include ::Gitlab::Geo::ReplicableModel + with_replicator Geo::TerraformStateVersionReplicator scope :project_id_in, ->(ids) { joins(:terraform_state).where('terraform_states.project_id': ids) } @@ -36,11 +37,11 @@ def selective_sync_scope(node) project_id_in(node.projects) end - end - def log_geo_deleted_event - # Keep empty for now. Should be addressed in future - # by https://gitlab.com/gitlab-org/gitlab/-/issues/232917 + def log_geo_deleted_event + # Keep empty for now. Should be addressed in future + # by https://gitlab.com/gitlab-org/gitlab/-/issues/232917 + end end end end diff --git a/ee/app/models/geo/pipeline_artifact_registry.rb b/ee/app/models/geo/pipeline_artifact_registry.rb new file mode 100644 index 0000000000000000000000000000000000000000..966023581890388c53f064c47673cf1b175ba598 --- /dev/null +++ b/ee/app/models/geo/pipeline_artifact_registry.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +module Geo + class PipelineArtifactRegistry < Geo::BaseRegistry + include ::Geo::ReplicableRegistry + include ::Geo::VerifiableRegistry + + MODEL_CLASS = ::Ci::PipelineArtifact + MODEL_FOREIGN_KEY = :pipeline_artifact_id + + belongs_to :pipeline_artifact, class_name: '::Ci::PipelineArtifact' + end +end diff --git a/ee/app/replicators/geo/pipeline_artifact_replicator.rb b/ee/app/replicators/geo/pipeline_artifact_replicator.rb new file mode 100644 index 0000000000000000000000000000000000000000..fe58f3dc84a59642c8a875f7fa8bee8f102ebd52 --- /dev/null +++ b/ee/app/replicators/geo/pipeline_artifact_replicator.rb @@ -0,0 +1,25 @@ +# frozen_string_literal: true + +module Geo + class PipelineArtifactReplicator < Gitlab::Geo::Replicator + include ::Geo::BlobReplicatorStrategy + extend ::Gitlab::Utils::Override + + def self.model + ::Ci::PipelineArtifact + end + + def carrierwave_uploader + model_record.file + end + + def self.replication_enabled_by_default? + false + end + + override :verification_feature_flag_enabled? + def self.verification_feature_flag_enabled? + Feature.enabled?(:geo_pipeline_artifact_replication, default_enabled: :yaml) + end + end +end diff --git a/ee/app/workers/geo/secondary/registry_consistency_worker.rb b/ee/app/workers/geo/secondary/registry_consistency_worker.rb index 667c0dbf6b5c2ff6502be1d43327c31743ca5410..fe2333416ea1a0b9b3ddba9bf8043c312726b8d4 100644 --- a/ee/app/workers/geo/secondary/registry_consistency_worker.rb +++ b/ee/app/workers/geo/secondary/registry_consistency_worker.rb @@ -26,7 +26,8 @@ class RegistryConsistencyWorker Geo::TerraformStateVersionRegistry, Geo::UploadRegistry, Geo::SnippetRepositoryRegistry, - Geo::GroupWikiRepositoryRegistry + Geo::GroupWikiRepositoryRegistry, + Geo::PipelineArtifactRegistry ].freeze BATCH_SIZE = 10000 diff --git a/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb b/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb new file mode 100644 index 0000000000000000000000000000000000000000..3c7058878b4f50ddfa21d970a6faf788cc97f728 --- /dev/null +++ b/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true + +class CreatePipelineArtifactRegistry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + create_table :pipeline_artifact_registry, id: :bigserial, force: :cascade do |t| + t.bigint :pipeline_artifact_id, null: false + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :last_synced_at + t.datetime_with_timezone :retry_at + t.datetime_with_timezone :verified_at + t.datetime_with_timezone :verification_started_at + t.datetime_with_timezone :verification_retry_at + t.integer :state, default: 0, null: false, limit: 2 + t.integer :verification_state, default: 0, null: false, limit: 2 + t.integer :retry_count, default: 0, limit: 2 + t.integer :verification_retry_count, default: 0, limit: 2 + t.boolean :checksum_mismatch + t.binary :verification_checksum + t.binary :verification_checksum_mismatched + t.string :verification_failure, limit: 255 # rubocop:disable Migration/PreventStrings + t.string :last_sync_failure, limit: 255 # rubocop:disable Migration/PreventStrings + + t.index :pipeline_artifact_id, name: :index_pipeline_artifact_registry_on_pipeline_artifact_id, unique: true + t.index :retry_at + t.index :state + t.index :verification_retry_at, name: :pipeline_artifact_registry_failed_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + t.index :verification_state, name: :pipeline_artifact_registry_needs_verification, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + t.index :verified_at, name: :pipeline_artifact_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" + end + end + + def down + drop_table :pipeline_artifact_registry + end +end diff --git a/ee/db/geo/schema.rb b/ee/db/geo/schema.rb index 81a1037b24f1300aa06ef1304ed7b86b697299a2..bd43b932a8fcdb078e1e21ee7b57ac7c73423104 100644 --- a/ee/db/geo/schema.rb +++ b/ee/db/geo/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_02_25_200858) do +ActiveRecord::Schema.define(version: 2021_03_11_204306) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -148,6 +148,31 @@ t.index ["verified_at"], name: "package_file_registry_pending_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" end + create_table "pipeline_artifact_registry", force: :cascade do |t| + t.bigint "pipeline_artifact_id", null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "last_synced_at" + t.datetime_with_timezone "retry_at" + t.datetime_with_timezone "verified_at" + t.datetime_with_timezone "verification_started_at" + t.datetime_with_timezone "verification_retry_at" + t.integer "state", limit: 2, default: 0, null: false + t.integer "verification_state", limit: 2, default: 0, null: false + t.integer "retry_count", limit: 2, default: 0 + t.integer "verification_retry_count", limit: 2, default: 0 + t.boolean "checksum_mismatch" + t.binary "verification_checksum" + t.binary "verification_checksum_mismatched" + t.string "verification_failure", limit: 255 + t.string "last_sync_failure", limit: 255 + t.index ["pipeline_artifact_id"], name: "index_pipeline_artifact_registry_on_pipeline_artifact_id", unique: true + t.index ["retry_at"], name: "index_pipeline_artifact_registry_on_retry_at" + t.index ["state"], name: "index_pipeline_artifact_registry_on_state" + t.index ["verification_retry_at"], name: "pipeline_artifact_registry_failed_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + t.index ["verification_state"], name: "pipeline_artifact_registry_needs_verification", where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + t.index ["verified_at"], name: "pipeline_artifact_registry_pending_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" + end + create_table "project_registry", id: :serial, force: :cascade do |t| t.integer "project_id", null: false t.datetime "last_repository_synced_at" diff --git a/ee/lib/gitlab/geo.rb b/ee/lib/gitlab/geo.rb index f5ff1e24eb8bc18250306798af5c37920e10d688..dc11e8c66a3445b2d5e0312e61a2fc2d8834c72e 100644 --- a/ee/lib/gitlab/geo.rb +++ b/ee/lib/gitlab/geo.rb @@ -24,7 +24,8 @@ module Geo ::Geo::PackageFileReplicator, ::Geo::TerraformStateVersionReplicator, ::Geo::SnippetRepositoryReplicator, - ::Geo::GroupWikiRepositoryReplicator + ::Geo::GroupWikiRepositoryReplicator, + ::Geo::PipelineArtifactReplicator ].freeze def self.current_node diff --git a/ee/spec/factories/geo/pipeline_artifact_registry.rb b/ee/spec/factories/geo/pipeline_artifact_registry.rb new file mode 100644 index 0000000000000000000000000000000000000000..f134af239434d2dee6e96ff14ed52c2baec8584f --- /dev/null +++ b/ee/spec/factories/geo/pipeline_artifact_registry.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :geo_pipeline_artifact_registry, class: 'Geo::PipelineArtifactRegistry' do + pipeline_artifact factory: :ci_pipeline_artifact + state { Geo::PipelineArtifactRegistry.state_value(:pending) } + + trait :synced do + state { Geo::PipelineArtifactRegistry.state_value(:synced) } + last_synced_at { 5.days.ago } + end + + trait :failed do + state { Geo::PipelineArtifactRegistry.state_value(:failed) } + last_synced_at { 1.day.ago } + retry_count { 2 } + last_sync_failure { 'Random error' } + end + + trait :started do + state { Geo::PipelineArtifactRegistry.state_value(:started) } + last_synced_at { 1.day.ago } + retry_count { 0 } + end + end +end diff --git a/ee/spec/finders/geo/pipeline_artifact_registry_finder_spec.rb b/ee/spec/finders/geo/pipeline_artifact_registry_finder_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..db8318c69c08d5574010a57b1a513a59c4b8779e --- /dev/null +++ b/ee/spec/finders/geo/pipeline_artifact_registry_finder_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::PipelineArtifactRegistryFinder do + it_behaves_like 'a framework registry finder', :geo_pipeline_artifact_registry +end diff --git a/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json b/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json index 4b8696edfea395e828f82ecf22f9531866312837..763a679a6bd891388ca4d33c4a257e32e2639d5f 100644 --- a/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json +++ b/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json @@ -108,6 +108,18 @@ "group_wiki_repositories_verification_failed_count", "group_wiki_repositories_synced_in_percentage", "group_wiki_repositories_verified_in_percentage", + "pipeline_artifacts_count", + "pipeline_artifacts_checksum_total_count", + "pipeline_artifacts_checksummed_count", + "pipeline_artifacts_checksum_failed_count", + "pipeline_artifacts_synced_count", + "pipeline_artifacts_failed_count", + "pipeline_artifacts_registry_count", + "pipeline_artifacts_verification_total_count", + "pipeline_artifacts_verified_count", + "pipeline_artifacts_verification_failed_count", + "pipeline_artifacts_synced_in_percentage", + "pipeline_artifacts_verified_in_percentage", "repositories_verified_count", "repositories_verification_failed_count", "repositories_verification_total_count", @@ -254,6 +266,18 @@ "group_wiki_repositories_verification_total_count": { "type": ["integer", "null"] }, "group_wiki_repositories_verified_count": { "type": ["integer", "null"] }, "group_wiki_repositories_verified_in_percentage": { "type": "string" }, + "pipeline_artifacts_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_checksummed_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_checksum_failed_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_checksum_total_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_registry_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_failed_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_synced_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_synced_in_percentage": { "type": "string" }, + "pipeline_artifacts_verification_failed_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_verification_total_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_verified_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_verified_in_percentage": { "type": "string" }, "repositories_verified_count": { "type": ["integer", "null"] }, "repositories_verification_failed_count": { "type": ["integer", "null"] }, "repositories_verification_total_count": { "type": ["integer", "null"] }, diff --git a/ee/spec/graphql/resolvers/geo/pipeline_artifact_registries_resolver_spec.rb b/ee/spec/graphql/resolvers/geo/pipeline_artifact_registries_resolver_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..dd6cacb90a88af264ddbec1d224c9ec37d20d227 --- /dev/null +++ b/ee/spec/graphql/resolvers/geo/pipeline_artifact_registries_resolver_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Resolvers::Geo::PipelineArtifactRegistriesResolver do + it_behaves_like 'a Geo registries resolver', :geo_pipeline_artifact_registry +end diff --git a/ee/spec/graphql/types/geo/geo_node_type_spec.rb b/ee/spec/graphql/types/geo/geo_node_type_spec.rb index 2f404f57eff5c7cf0a63b221afee5dcaeac63151..2f25660175a226562b5fc9065defe878376752f6 100644 --- a/ee/spec/graphql/types/geo/geo_node_type_spec.rb +++ b/ee/spec/graphql/types/geo/geo_node_type_spec.rb @@ -14,6 +14,7 @@ minimum_reverification_interval merge_request_diff_registries package_file_registries snippet_repository_registries terraform_state_version_registries group_wiki_repository_registries + pipeline_artifact_registries ] expect(described_class).to have_graphql_fields(*expected_fields) diff --git a/ee/spec/graphql/types/geo/pipeline_artifact_registry_type_spec.rb b/ee/spec/graphql/types/geo/pipeline_artifact_registry_type_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..f8c02829a24af5eb6266121f4f034168ba8395cc --- /dev/null +++ b/ee/spec/graphql/types/geo/pipeline_artifact_registry_type_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe GitlabSchema.types['PipelineArtifactRegistry'] do + it_behaves_like 'a Geo registry type' + + it 'has the expected fields (other than those included in RegistryType)' do + expected_fields = %i[pipeline_artifact_id] + + expect(described_class).to have_graphql_fields(*expected_fields).at_least + end +end diff --git a/ee/spec/models/geo/pipeline_artifact_registry_spec.rb b/ee/spec/models/geo/pipeline_artifact_registry_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..93c3cbaac9958022ef95c9cfb47e168f8af7117e --- /dev/null +++ b/ee/spec/models/geo/pipeline_artifact_registry_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::PipelineArtifactRegistry, :geo, type: :model do + let_it_be(:registry) { create(:geo_pipeline_artifact_registry) } + + specify 'factory is valid' do + expect(registry).to be_valid + end + + include_examples 'a Geo framework registry' +end diff --git a/ee/spec/replicators/geo/pipeline_artifact_replicator_spec.rb b/ee/spec/replicators/geo/pipeline_artifact_replicator_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..0e3e4afa6ea0fc399a594f49f705daa7b732830d --- /dev/null +++ b/ee/spec/replicators/geo/pipeline_artifact_replicator_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::PipelineArtifactReplicator do + let(:model_record) { build(:ci_pipeline_artifact, :with_coverage_report) } + + include_examples 'a blob replicator' +end diff --git a/ee/spec/requests/api/graphql/geo/registries_spec.rb b/ee/spec/requests/api/graphql/geo/registries_spec.rb index af8feca99798cb453cf840cdac874d486a3479de..217417e51d9296034763680286623a170779e1f4 100644 --- a/ee/spec/requests/api/graphql/geo/registries_spec.rb +++ b/ee/spec/requests/api/graphql/geo/registries_spec.rb @@ -37,4 +37,11 @@ registry_factory: :geo_group_wiki_repository_registry, registry_foreign_key_field_name: 'groupWikiRepositoryId' } + + it_behaves_like 'gets registries for', { + field_name: 'pipelineArtifactRegistries', + registry_class_name: 'PipelineArtifactRegistry', + registry_factory: :geo_pipeline_artifact_registry, + registry_foreign_key_field_name: 'pipelineArtifactId' + } end diff --git a/spec/models/concerns/ci/artifactable_spec.rb b/spec/models/concerns/ci/artifactable_spec.rb index ebc838e86a6c6d310f14a3171fdebcc0161ea907..d364e3e848d67d5cc6331d76efb36be61479fd77 100644 --- a/spec/models/concerns/ci/artifactable_spec.rb +++ b/spec/models/concerns/ci/artifactable_spec.rb @@ -72,5 +72,11 @@ expect(Ci::JobArtifact.expired(1).order_id_asc).to eq([recently_expired_artifact]) end end + + describe '.with_files_stored_locally' do + it 'returns artifact stored locally' do + expect(Ci::JobArtifact.with_files_stored_locally).to contain_exactly(recently_expired_artifact, later_expired_artifact, not_expired_artifact) + end + end end end