From fd586be1fc2d8a767c77b8477bccd124e05486c9 Mon Sep 17 00:00:00 2001 From: Maxime Orefice Date: Thu, 11 Mar 2021 15:59:38 -0500 Subject: [PATCH 1/6] Add pipeline artifact to geo replication POC --- app/models/ci/pipeline_artifact.rb | 2 + .../geo_pipeline_artifact_replication.yml | 8 +++ .../000_inflections.rb | 1 + ee/app/models/ee/ci/pipeline_artifact.rb | 53 +++++++++++++++++++ .../models/geo/pipeline_artifact_registry.rb | 14 +++++ .../geo/pipeline_artifact_replicator.rb | 19 +++++++ .../secondary/registry_consistency_worker.rb | 3 +- ...04306_create_pipeline_artifact_registry.rb | 43 +++++++++++++++ ee/db/geo/schema.rb | 27 +++++++++- ee/lib/gitlab/geo.rb | 3 +- .../geo/pipeline_artifact_registry.rb | 26 +++++++++ .../geo/pipeline_artifact_registry_spec.rb | 13 +++++ .../geo/pipeline_artifact_replicator_spec.rb | 9 ++++ 13 files changed, 218 insertions(+), 3 deletions(-) create mode 100644 config/feature_flags/development/geo_pipeline_artifact_replication.yml create mode 100644 ee/app/models/ee/ci/pipeline_artifact.rb create mode 100644 ee/app/models/geo/pipeline_artifact_registry.rb create mode 100644 ee/app/replicators/geo/pipeline_artifact_replicator.rb create mode 100644 ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb create mode 100644 ee/spec/factories/geo/pipeline_artifact_registry.rb create mode 100644 ee/spec/models/geo/pipeline_artifact_registry_spec.rb create mode 100644 ee/spec/replicators/geo/pipeline_artifact_replicator_spec.rb diff --git a/app/models/ci/pipeline_artifact.rb b/app/models/ci/pipeline_artifact.rb index f538a4cd808954..17c47318146466 100644 --- a/app/models/ci/pipeline_artifact.rb +++ b/app/models/ci/pipeline_artifact.rb @@ -57,3 +57,5 @@ def present end end end + +Ci::PipelineArtifact.prepend_if_ee('::EE::Ci::PipelineArtifact') diff --git a/config/feature_flags/development/geo_pipeline_artifact_replication.yml b/config/feature_flags/development/geo_pipeline_artifact_replication.yml new file mode 100644 index 00000000000000..f767a005a0dbdc --- /dev/null +++ b/config/feature_flags/development/geo_pipeline_artifact_replication.yml @@ -0,0 +1,8 @@ +--- +name: geo_pipeline_artifact_replication +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/238464 +rollout_issue_url: +milestone: '13.10' +type: development +group: group::testing +default_enabled: false diff --git a/config/initializers_before_autoloader/000_inflections.rb b/config/initializers_before_autoloader/000_inflections.rb index 308b38c1ba249d..93cf9e01b96f38 100644 --- a/config/initializers_before_autoloader/000_inflections.rb +++ b/config/initializers_before_autoloader/000_inflections.rb @@ -24,6 +24,7 @@ job_artifact_registry lfs_object_registry package_file_registry + pipeline_artifact_registry project_auto_devops project_registry project_statistics diff --git a/ee/app/models/ee/ci/pipeline_artifact.rb b/ee/app/models/ee/ci/pipeline_artifact.rb new file mode 100644 index 00000000000000..3fc52841e69082 --- /dev/null +++ b/ee/app/models/ee/ci/pipeline_artifact.rb @@ -0,0 +1,53 @@ +# frozen_string_literal: true + +module EE + module Ci + module PipelineArtifact + extend ActiveSupport::Concern + + prepended do + after_destroy :log_geo_deleted_event + + include ::Gitlab::Geo::ReplicableModel + + with_replicator ::Geo::PipelineArtifactReplicator + + def self.replicables_for_current_secondary(primary_key_in) + node = ::Gitlab::Geo.current_node + + primary_key_in(primary_key_in) + .merge(selective_sync_scope(node)) + .merge(object_storage_scope(node)) + end + end + + class_methods do + # @param primary_key_in [Range, Ci::JobArtifact] arg to pass to primary_key_in scope + # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key + def replicables_for_current_secondary(primary_key_in) + node = ::Gitlab::Geo.current_node + + primary_key_in(primary_key_in) + .merge(selective_sync_scope(node)) + .merge(object_storage_scope(node)) + end + + def object_storage_scope(node) + return all if node.sync_object_storage? + + with_files_stored_locally + end + + def selective_sync_scope(node) + return all unless node.selective_sync? + + project_id_in(node.projects) + end + end + + def log_geo_deleted_event + ::Geo::JobArtifactDeletedEventStore.new(self).create! + end + end + end +end diff --git a/ee/app/models/geo/pipeline_artifact_registry.rb b/ee/app/models/geo/pipeline_artifact_registry.rb new file mode 100644 index 00000000000000..0b967bf4889074 --- /dev/null +++ b/ee/app/models/geo/pipeline_artifact_registry.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +module Geo + class PipelineArtifactRegistry < Geo::BaseRegistry + include Geo::ReplicableRegistry + include ::Gitlab::Geo::VerificationState + include ::Geo::VerifiableRegistry + + MODEL_CLASS = ::Ci::PipelineArtifact + MODEL_FOREIGN_KEY = :pipeline_artifact_id + + belongs_to :pipeline_artifact, class_name: '::Ci::PipelineArtifact' + end +end diff --git a/ee/app/replicators/geo/pipeline_artifact_replicator.rb b/ee/app/replicators/geo/pipeline_artifact_replicator.rb new file mode 100644 index 00000000000000..78c99fe49cd621 --- /dev/null +++ b/ee/app/replicators/geo/pipeline_artifact_replicator.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +module Geo + class PipelineArtifactReplicator < Gitlab::Geo::Replicator + include ::Geo::BlobReplicatorStrategy + + def self.model + ::Ci::PipelineArtifact + end + + def self.replication_enabled_by_default? + false + end + + def carrierwave_uploader + model_record.file + end + end +end diff --git a/ee/app/workers/geo/secondary/registry_consistency_worker.rb b/ee/app/workers/geo/secondary/registry_consistency_worker.rb index 667c0dbf6b5c2f..fe2333416ea1a0 100644 --- a/ee/app/workers/geo/secondary/registry_consistency_worker.rb +++ b/ee/app/workers/geo/secondary/registry_consistency_worker.rb @@ -26,7 +26,8 @@ class RegistryConsistencyWorker Geo::TerraformStateVersionRegistry, Geo::UploadRegistry, Geo::SnippetRepositoryRegistry, - Geo::GroupWikiRepositoryRegistry + Geo::GroupWikiRepositoryRegistry, + Geo::PipelineArtifactRegistry ].freeze BATCH_SIZE = 10000 diff --git a/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb b/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb new file mode 100644 index 00000000000000..cb8eaec519b4de --- /dev/null +++ b/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +class CreatePipelineArtifactRegistry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + create_table :pipeline_artifact_registry, id: :bigserial, force: :cascade do |t| + t.bigint :pipeline_artifact_id, null: false + t.datetime_with_timezone :created_at, null: false + t.datetime_with_timezone :last_synced_at + t.datetime_with_timezone :retry_at + t.datetime_with_timezone :verified_at + t.datetime_with_timezone :verification_started_at + t.datetime_with_timezone :verification_retry_at + t.integer :state, default: 0, null: false, limit: 2 + t.integer :verification_state, default: 0, null: false, limit: 2 + t.integer :retry_count, default: 0, limit: 2 + t.integer :verification_retry_count, default: 0, limit: 2 + t.boolean :checksum_mismatch + t.binary :verification_checksum + t.binary :verification_checksum_mismatched + t.text :verification_failure + t.text :last_sync_failure + + t.index :pipeline_artifact_id, name: :index_pipeline_artifact_registry_on_pipeline_artifact_id, unique: true + t.index :retry_at + t.index :state + t.index :verification_retry_at, name: :pipeline_artifact_registry_failed_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + t.index :verification_state, name: :pipeline_artifact_registry_needs_verification, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + t.index :verified_at, name: :pipeline_artifact_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" + end + + add_text_limit :pipeline_artifact_registry, :verification_failure, 255 + add_text_limit :pipeline_artifact_registry, :last_sync_failure, 255 + end + + def down + end +end diff --git a/ee/db/geo/schema.rb b/ee/db/geo/schema.rb index 81a1037b24f130..c17f5334648692 100644 --- a/ee/db/geo/schema.rb +++ b/ee/db/geo/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_02_25_200858) do +ActiveRecord::Schema.define(version: 2021_03_11_204306) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -148,6 +148,31 @@ t.index ["verified_at"], name: "package_file_registry_pending_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" end + create_table "pipeline_artifact_registry", force: :cascade do |t| + t.bigint "pipeline_artifact_id", null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "last_synced_at" + t.datetime_with_timezone "retry_at" + t.datetime_with_timezone "verified_at" + t.datetime_with_timezone "verification_started_at" + t.datetime_with_timezone "verification_retry_at" + t.integer "state", limit: 2, default: 0, null: false + t.integer "verification_state", limit: 2, default: 0, null: false + t.integer "retry_count", limit: 2, default: 0 + t.integer "verification_retry_count", limit: 2, default: 0 + t.boolean "checksum_mismatch" + t.binary "verification_checksum" + t.binary "verification_checksum_mismatched" + t.text "verification_failure" + t.text "last_sync_failure" + t.index ["pipeline_artifact_id"], name: "index_pipeline_artifact_registry_on_pipeline_artifact_id", unique: true + t.index ["retry_at"], name: "index_pipeline_artifact_registry_on_retry_at" + t.index ["state"], name: "index_pipeline_artifact_registry_on_state" + t.index ["verification_retry_at"], name: "pipeline_artifact_registry_failed_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + t.index ["verification_state"], name: "pipeline_artifact_registry_needs_verification", where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + t.index ["verified_at"], name: "pipeline_artifact_registry_pending_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" + end + create_table "project_registry", id: :serial, force: :cascade do |t| t.integer "project_id", null: false t.datetime "last_repository_synced_at" diff --git a/ee/lib/gitlab/geo.rb b/ee/lib/gitlab/geo.rb index f5ff1e24eb8bc1..dc11e8c66a3445 100644 --- a/ee/lib/gitlab/geo.rb +++ b/ee/lib/gitlab/geo.rb @@ -24,7 +24,8 @@ module Geo ::Geo::PackageFileReplicator, ::Geo::TerraformStateVersionReplicator, ::Geo::SnippetRepositoryReplicator, - ::Geo::GroupWikiRepositoryReplicator + ::Geo::GroupWikiRepositoryReplicator, + ::Geo::PipelineArtifactReplicator ].freeze def self.current_node diff --git a/ee/spec/factories/geo/pipeline_artifact_registry.rb b/ee/spec/factories/geo/pipeline_artifact_registry.rb new file mode 100644 index 00000000000000..f134af239434d2 --- /dev/null +++ b/ee/spec/factories/geo/pipeline_artifact_registry.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :geo_pipeline_artifact_registry, class: 'Geo::PipelineArtifactRegistry' do + pipeline_artifact factory: :ci_pipeline_artifact + state { Geo::PipelineArtifactRegistry.state_value(:pending) } + + trait :synced do + state { Geo::PipelineArtifactRegistry.state_value(:synced) } + last_synced_at { 5.days.ago } + end + + trait :failed do + state { Geo::PipelineArtifactRegistry.state_value(:failed) } + last_synced_at { 1.day.ago } + retry_count { 2 } + last_sync_failure { 'Random error' } + end + + trait :started do + state { Geo::PipelineArtifactRegistry.state_value(:started) } + last_synced_at { 1.day.ago } + retry_count { 0 } + end + end +end diff --git a/ee/spec/models/geo/pipeline_artifact_registry_spec.rb b/ee/spec/models/geo/pipeline_artifact_registry_spec.rb new file mode 100644 index 00000000000000..93c3cbaac99580 --- /dev/null +++ b/ee/spec/models/geo/pipeline_artifact_registry_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::PipelineArtifactRegistry, :geo, type: :model do + let_it_be(:registry) { create(:geo_pipeline_artifact_registry) } + + specify 'factory is valid' do + expect(registry).to be_valid + end + + include_examples 'a Geo framework registry' +end diff --git a/ee/spec/replicators/geo/pipeline_artifact_replicator_spec.rb b/ee/spec/replicators/geo/pipeline_artifact_replicator_spec.rb new file mode 100644 index 00000000000000..0e3e4afa6ea0fc --- /dev/null +++ b/ee/spec/replicators/geo/pipeline_artifact_replicator_spec.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::PipelineArtifactReplicator do + let(:model_record) { build(:ci_pipeline_artifact, :with_coverage_report) } + + include_examples 'a blob replicator' +end -- GitLab From c91c90c1c45ce84c3cb29b3d56109bb31856fd35 Mon Sep 17 00:00:00 2001 From: Maxime Orefice Date: Mon, 22 Mar 2021 16:52:07 -0400 Subject: [PATCH 2/6] Use string over text columns --- .../20210311204306_create_pipeline_artifact_registry.rb | 8 +++----- ee/db/geo/schema.rb | 4 ++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb b/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb index cb8eaec519b4de..3c7058878b4f50 100644 --- a/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb +++ b/ee/db/geo/migrate/20210311204306_create_pipeline_artifact_registry.rb @@ -23,8 +23,8 @@ def up t.boolean :checksum_mismatch t.binary :verification_checksum t.binary :verification_checksum_mismatched - t.text :verification_failure - t.text :last_sync_failure + t.string :verification_failure, limit: 255 # rubocop:disable Migration/PreventStrings + t.string :last_sync_failure, limit: 255 # rubocop:disable Migration/PreventStrings t.index :pipeline_artifact_id, name: :index_pipeline_artifact_registry_on_pipeline_artifact_id, unique: true t.index :retry_at @@ -33,11 +33,9 @@ def up t.index :verification_state, name: :pipeline_artifact_registry_needs_verification, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" t.index :verified_at, name: :pipeline_artifact_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" end - - add_text_limit :pipeline_artifact_registry, :verification_failure, 255 - add_text_limit :pipeline_artifact_registry, :last_sync_failure, 255 end def down + drop_table :pipeline_artifact_registry end end diff --git a/ee/db/geo/schema.rb b/ee/db/geo/schema.rb index c17f5334648692..bd43b932a8fcdb 100644 --- a/ee/db/geo/schema.rb +++ b/ee/db/geo/schema.rb @@ -163,8 +163,8 @@ t.boolean "checksum_mismatch" t.binary "verification_checksum" t.binary "verification_checksum_mismatched" - t.text "verification_failure" - t.text "last_sync_failure" + t.string "verification_failure", limit: 255 + t.string "last_sync_failure", limit: 255 t.index ["pipeline_artifact_id"], name: "index_pipeline_artifact_registry_on_pipeline_artifact_id", unique: true t.index ["retry_at"], name: "index_pipeline_artifact_registry_on_retry_at" t.index ["state"], name: "index_pipeline_artifact_registry_on_state" -- GitLab From 36b59cdb78f3b0b7f2cb9595058b8c315b5ef51e Mon Sep 17 00:00:00 2001 From: Maxime Orefice Date: Mon, 22 Mar 2021 17:11:37 -0400 Subject: [PATCH 3/6] Add graphql api --- app/models/ci/job_artifact.rb | 1 - app/models/concerns/ci/artifactable.rb | 1 + .../mo-pipeline-artifact-geo-poc.yml | 5 +++ doc/api/graphql/reference/index.md | 35 +++++++++++++++++++ .../geo/pipeline_artifact_registry_finder.rb | 7 ++++ .../pipeline_artifact_registries_resolver.rb | 11 ++++++ ee/app/graphql/types/geo/geo_node_type.rb | 5 +++ .../geo/pipeline_artifact_registry_type.rb | 15 ++++++++ ee/app/models/ee/ci/pipeline_artifact.rb | 4 +-- .../pipeline_artifact_registry_finder_spec.rb | 7 ++++ .../public_api/v4/geo_node_status.json | 24 +++++++++++++ ...eline_artifact_registries_resolver_spec.rb | 7 ++++ .../graphql/types/geo/geo_node_type_spec.rb | 1 + .../pipeline_artifact_registry_type_spec.rb | 13 +++++++ .../api/graphql/geo/registries_spec.rb | 7 ++++ spec/models/concerns/ci/artifactable_spec.rb | 6 ++++ 16 files changed, 146 insertions(+), 3 deletions(-) create mode 100644 changelogs/unreleased/mo-pipeline-artifact-geo-poc.yml create mode 100644 ee/app/finders/geo/pipeline_artifact_registry_finder.rb create mode 100644 ee/app/graphql/resolvers/geo/pipeline_artifact_registries_resolver.rb create mode 100644 ee/app/graphql/types/geo/pipeline_artifact_registry_type.rb create mode 100644 ee/spec/finders/geo/pipeline_artifact_registry_finder_spec.rb create mode 100644 ee/spec/graphql/resolvers/geo/pipeline_artifact_registries_resolver_spec.rb create mode 100644 ee/spec/graphql/types/geo/pipeline_artifact_registry_type_spec.rb diff --git a/app/models/ci/job_artifact.rb b/app/models/ci/job_artifact.rb index d5e88f2be5bea1..705e377f66547f 100644 --- a/app/models/ci/job_artifact.rb +++ b/app/models/ci/job_artifact.rb @@ -131,7 +131,6 @@ class JobArtifact < ApplicationRecord update_project_statistics project_statistics_name: :build_artifacts_size scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } - scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) } scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :for_sha, ->(sha, project_id) { joins(job: :pipeline).where(ci_pipelines: { sha: sha, project_id: project_id }) } scope :for_job_name, ->(name) { joins(:job).where(ci_builds: { name: name }) } diff --git a/app/models/concerns/ci/artifactable.rb b/app/models/concerns/ci/artifactable.rb index cbe7d3b6abb1e8..5ae3d5d10c11dd 100644 --- a/app/models/concerns/ci/artifactable.rb +++ b/app/models/concerns/ci/artifactable.rb @@ -20,6 +20,7 @@ module Artifactable scope :expired_before, -> (timestamp) { where(arel_table[:expire_at].lt(timestamp)) } scope :expired, -> (limit) { expired_before(Time.current).limit(limit) } + scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) } end def each_blob(&blk) diff --git a/changelogs/unreleased/mo-pipeline-artifact-geo-poc.yml b/changelogs/unreleased/mo-pipeline-artifact-geo-poc.yml new file mode 100644 index 00000000000000..97661f99482fee --- /dev/null +++ b/changelogs/unreleased/mo-pipeline-artifact-geo-poc.yml @@ -0,0 +1,5 @@ +--- +title: Include pipeline artifacts in Geo replication +merge_request: 56423 +author: +type: added diff --git a/doc/api/graphql/reference/index.md b/doc/api/graphql/reference/index.md index 331d3be2b8c3c9..95f003d7db4daa 100644 --- a/doc/api/graphql/reference/index.md +++ b/doc/api/graphql/reference/index.md @@ -3016,6 +3016,7 @@ Represents an external issue. | `minimumReverificationInterval` | [`Int`](#int) | The interval (in days) in which the repository verification is valid. Once expired, it will be reverified. | | `name` | [`String`](#string) | The unique identifier for this Geo node. | | `packageFileRegistries` | [`PackageFileRegistryConnection`](#packagefileregistryconnection) | Package file registries of the GeoNode. | +| `pipelineArtifactRegistries` | [`PipelineArtifactRegistryConnection`](#pipelineartifactregistryconnection) | Find pipeline artifact registries on this Geo node. Available only when feature flag `geo_pipeline_artifact_replication` is enabled. | | `primary` | [`Boolean`](#boolean) | Indicates whether this Geo node is the primary. | | `reposMaxCapacity` | [`Int`](#int) | The maximum concurrency of repository backfill for this secondary node. | | `selectiveSyncNamespaces` | [`NamespaceConnection`](#namespaceconnection) | The namespaces that should be synced, if `selective_sync_type` == `namespaces`. | @@ -4624,6 +4625,40 @@ Information about pagination in a connection. | `yearPipelinesSuccessful` | [`[Int!]`](#int) | Total yearly successful pipeline count. | | `yearPipelinesTotals` | [`[Int!]`](#int) | Total yearly pipeline count. | +### `PipelineArtifactRegistry` + +Represents the Geo sync and verification state of a pipeline artifact. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `createdAt` | [`Time`](#time) | Timestamp when the PipelineArtifactRegistry was created | +| `id` | [`ID!`](#id) | ID of the PipelineArtifactRegistry | +| `lastSyncFailure` | [`String`](#string) | Error message during sync of the PipelineArtifactRegistry | +| `lastSyncedAt` | [`Time`](#time) | Timestamp of the most recent successful sync of the PipelineArtifactRegistry | +| `pipelineArtifactId` | [`ID!`](#id) | ID of the pipeline artifact. | +| `retryAt` | [`Time`](#time) | Timestamp after which the PipelineArtifactRegistry should be resynced | +| `retryCount` | [`Int`](#int) | Number of consecutive failed sync attempts of the PipelineArtifactRegistry | +| `state` | [`RegistryState`](#registrystate) | Sync state of the PipelineArtifactRegistry | + +### `PipelineArtifactRegistryConnection` + +The connection type for PipelineArtifactRegistry. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `edges` | [`[PipelineArtifactRegistryEdge]`](#pipelineartifactregistryedge) | A list of edges. | +| `nodes` | [`[PipelineArtifactRegistry]`](#pipelineartifactregistry) | A list of nodes. | +| `pageInfo` | [`PageInfo!`](#pageinfo) | Information to aid in pagination. | + +### `PipelineArtifactRegistryEdge` + +An edge in a connection. + +| Field | Type | Description | +| ----- | ---- | ----------- | +| `cursor` | [`String!`](#string) | A cursor for use in pagination. | +| `node` | [`PipelineArtifactRegistry`](#pipelineartifactregistry) | The item at the end of the edge. | + ### `PipelineCancelPayload` Autogenerated return type of PipelineCancel. diff --git a/ee/app/finders/geo/pipeline_artifact_registry_finder.rb b/ee/app/finders/geo/pipeline_artifact_registry_finder.rb new file mode 100644 index 00000000000000..ec68f5248be129 --- /dev/null +++ b/ee/app/finders/geo/pipeline_artifact_registry_finder.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module Geo + class PipelineArtifactRegistryFinder + include FrameworkRegistryFinder + end +end diff --git a/ee/app/graphql/resolvers/geo/pipeline_artifact_registries_resolver.rb b/ee/app/graphql/resolvers/geo/pipeline_artifact_registries_resolver.rb new file mode 100644 index 00000000000000..61c51d75ae9124 --- /dev/null +++ b/ee/app/graphql/resolvers/geo/pipeline_artifact_registries_resolver.rb @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +module Resolvers + module Geo + class PipelineArtifactRegistriesResolver < BaseResolver + type ::Types::Geo::GeoNodeType.connection_type, null: true + + include RegistriesResolver + end + end +end diff --git a/ee/app/graphql/types/geo/geo_node_type.rb b/ee/app/graphql/types/geo/geo_node_type.rb index eba46fe195aa71..82456ce4651b5e 100644 --- a/ee/app/graphql/types/geo/geo_node_type.rb +++ b/ee/app/graphql/types/geo/geo_node_type.rb @@ -42,6 +42,11 @@ class GeoNodeType < BaseObject null: true, resolver: ::Resolvers::Geo::GroupWikiRepositoryRegistriesResolver, description: 'Find group wiki repository registries on this Geo node.' + field :pipeline_artifact_registries, ::Types::Geo::PipelineArtifactRegistryType.connection_type, + null: true, + resolver: ::Resolvers::Geo::PipelineArtifactRegistriesResolver, + description: 'Find pipeline artifact registries on this Geo node.', + feature_flag: :geo_pipeline_artifact_replication end end end diff --git a/ee/app/graphql/types/geo/pipeline_artifact_registry_type.rb b/ee/app/graphql/types/geo/pipeline_artifact_registry_type.rb new file mode 100644 index 00000000000000..1036d51fb06eba --- /dev/null +++ b/ee/app/graphql/types/geo/pipeline_artifact_registry_type.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module Types + module Geo + # rubocop:disable Graphql/AuthorizeTypes because it is included + class PipelineArtifactRegistryType < BaseObject + include ::Types::Geo::RegistryType + + graphql_name 'PipelineArtifactRegistry' + description 'Represents the Geo sync and verification state of a pipeline artifact' + + field :pipeline_artifact_id, GraphQL::ID_TYPE, null: false, description: 'ID of the pipeline artifact.' + end + end +end diff --git a/ee/app/models/ee/ci/pipeline_artifact.rb b/ee/app/models/ee/ci/pipeline_artifact.rb index 3fc52841e69082..835003452b8c98 100644 --- a/ee/app/models/ee/ci/pipeline_artifact.rb +++ b/ee/app/models/ee/ci/pipeline_artifact.rb @@ -22,8 +22,8 @@ def self.replicables_for_current_secondary(primary_key_in) end class_methods do - # @param primary_key_in [Range, Ci::JobArtifact] arg to pass to primary_key_in scope - # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key + # @param primary_key_in [Range, Ci::PipelineArtifact] arg to pass to primary_key_in scope + # @return [ActiveRecord::Relation] everything that should be synced to this node, restricted by primary key def replicables_for_current_secondary(primary_key_in) node = ::Gitlab::Geo.current_node diff --git a/ee/spec/finders/geo/pipeline_artifact_registry_finder_spec.rb b/ee/spec/finders/geo/pipeline_artifact_registry_finder_spec.rb new file mode 100644 index 00000000000000..db8318c69c08d5 --- /dev/null +++ b/ee/spec/finders/geo/pipeline_artifact_registry_finder_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Geo::PipelineArtifactRegistryFinder do + it_behaves_like 'a framework registry finder', :geo_pipeline_artifact_registry +end diff --git a/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json b/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json index 4b8696edfea395..763a679a6bd891 100644 --- a/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json +++ b/ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json @@ -108,6 +108,18 @@ "group_wiki_repositories_verification_failed_count", "group_wiki_repositories_synced_in_percentage", "group_wiki_repositories_verified_in_percentage", + "pipeline_artifacts_count", + "pipeline_artifacts_checksum_total_count", + "pipeline_artifacts_checksummed_count", + "pipeline_artifacts_checksum_failed_count", + "pipeline_artifacts_synced_count", + "pipeline_artifacts_failed_count", + "pipeline_artifacts_registry_count", + "pipeline_artifacts_verification_total_count", + "pipeline_artifacts_verified_count", + "pipeline_artifacts_verification_failed_count", + "pipeline_artifacts_synced_in_percentage", + "pipeline_artifacts_verified_in_percentage", "repositories_verified_count", "repositories_verification_failed_count", "repositories_verification_total_count", @@ -254,6 +266,18 @@ "group_wiki_repositories_verification_total_count": { "type": ["integer", "null"] }, "group_wiki_repositories_verified_count": { "type": ["integer", "null"] }, "group_wiki_repositories_verified_in_percentage": { "type": "string" }, + "pipeline_artifacts_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_checksummed_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_checksum_failed_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_checksum_total_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_registry_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_failed_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_synced_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_synced_in_percentage": { "type": "string" }, + "pipeline_artifacts_verification_failed_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_verification_total_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_verified_count": { "type": ["integer", "null"] }, + "pipeline_artifacts_verified_in_percentage": { "type": "string" }, "repositories_verified_count": { "type": ["integer", "null"] }, "repositories_verification_failed_count": { "type": ["integer", "null"] }, "repositories_verification_total_count": { "type": ["integer", "null"] }, diff --git a/ee/spec/graphql/resolvers/geo/pipeline_artifact_registries_resolver_spec.rb b/ee/spec/graphql/resolvers/geo/pipeline_artifact_registries_resolver_spec.rb new file mode 100644 index 00000000000000..dd6cacb90a88af --- /dev/null +++ b/ee/spec/graphql/resolvers/geo/pipeline_artifact_registries_resolver_spec.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Resolvers::Geo::PipelineArtifactRegistriesResolver do + it_behaves_like 'a Geo registries resolver', :geo_pipeline_artifact_registry +end diff --git a/ee/spec/graphql/types/geo/geo_node_type_spec.rb b/ee/spec/graphql/types/geo/geo_node_type_spec.rb index 2f404f57eff5c7..2f25660175a226 100644 --- a/ee/spec/graphql/types/geo/geo_node_type_spec.rb +++ b/ee/spec/graphql/types/geo/geo_node_type_spec.rb @@ -14,6 +14,7 @@ minimum_reverification_interval merge_request_diff_registries package_file_registries snippet_repository_registries terraform_state_version_registries group_wiki_repository_registries + pipeline_artifact_registries ] expect(described_class).to have_graphql_fields(*expected_fields) diff --git a/ee/spec/graphql/types/geo/pipeline_artifact_registry_type_spec.rb b/ee/spec/graphql/types/geo/pipeline_artifact_registry_type_spec.rb new file mode 100644 index 00000000000000..f8c02829a24af5 --- /dev/null +++ b/ee/spec/graphql/types/geo/pipeline_artifact_registry_type_spec.rb @@ -0,0 +1,13 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe GitlabSchema.types['PipelineArtifactRegistry'] do + it_behaves_like 'a Geo registry type' + + it 'has the expected fields (other than those included in RegistryType)' do + expected_fields = %i[pipeline_artifact_id] + + expect(described_class).to have_graphql_fields(*expected_fields).at_least + end +end diff --git a/ee/spec/requests/api/graphql/geo/registries_spec.rb b/ee/spec/requests/api/graphql/geo/registries_spec.rb index af8feca99798cb..217417e51d9296 100644 --- a/ee/spec/requests/api/graphql/geo/registries_spec.rb +++ b/ee/spec/requests/api/graphql/geo/registries_spec.rb @@ -37,4 +37,11 @@ registry_factory: :geo_group_wiki_repository_registry, registry_foreign_key_field_name: 'groupWikiRepositoryId' } + + it_behaves_like 'gets registries for', { + field_name: 'pipelineArtifactRegistries', + registry_class_name: 'PipelineArtifactRegistry', + registry_factory: :geo_pipeline_artifact_registry, + registry_foreign_key_field_name: 'pipelineArtifactId' + } end diff --git a/spec/models/concerns/ci/artifactable_spec.rb b/spec/models/concerns/ci/artifactable_spec.rb index ebc838e86a6c6d..d364e3e848d67d 100644 --- a/spec/models/concerns/ci/artifactable_spec.rb +++ b/spec/models/concerns/ci/artifactable_spec.rb @@ -72,5 +72,11 @@ expect(Ci::JobArtifact.expired(1).order_id_asc).to eq([recently_expired_artifact]) end end + + describe '.with_files_stored_locally' do + it 'returns artifact stored locally' do + expect(Ci::JobArtifact.with_files_stored_locally).to contain_exactly(recently_expired_artifact, later_expired_artifact, not_expired_artifact) + end + end end end -- GitLab From 8319222513ec329815ad248c1a965ed038a66af9 Mon Sep 17 00:00:00 2001 From: Maxime Orefice Date: Tue, 23 Mar 2021 14:18:54 -0400 Subject: [PATCH 4/6] Refactor some unused methods --- app/models/concerns/ci/artifactable.rb | 3 ++- .../geo_pipeline_artifact_replication.yml | 2 +- ee/app/models/ee/ci/job_artifact.rb | 1 - ee/app/models/ee/ci/pipeline_artifact.rb | 12 +----------- ee/app/models/ee/terraform/state_version.rb | 6 +----- ee/app/models/geo/pipeline_artifact_registry.rb | 2 -- .../replicators/geo/pipeline_artifact_replicator.rb | 10 ++++++---- 7 files changed, 11 insertions(+), 25 deletions(-) diff --git a/app/models/concerns/ci/artifactable.rb b/app/models/concerns/ci/artifactable.rb index 5ae3d5d10c11dd..e7e733d47ec04d 100644 --- a/app/models/concerns/ci/artifactable.rb +++ b/app/models/concerns/ci/artifactable.rb @@ -20,7 +20,8 @@ module Artifactable scope :expired_before, -> (timestamp) { where(arel_table[:expire_at].lt(timestamp)) } scope :expired, -> (limit) { expired_before(Time.current).limit(limit) } - scope :with_files_stored_locally, -> { where(file_store: ::JobArtifactUploader::Store::LOCAL) } + scope :with_files_stored_locally, -> { where(file_store: ::ObjectStorage::Store::LOCAL) } + scope :project_id_in, ->(ids) { where(project_id: ids) } end def each_blob(&blk) diff --git a/config/feature_flags/development/geo_pipeline_artifact_replication.yml b/config/feature_flags/development/geo_pipeline_artifact_replication.yml index f767a005a0dbdc..c8aa0e3effb653 100644 --- a/config/feature_flags/development/geo_pipeline_artifact_replication.yml +++ b/config/feature_flags/development/geo_pipeline_artifact_replication.yml @@ -4,5 +4,5 @@ introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/issues/238464 rollout_issue_url: milestone: '13.10' type: development -group: group::testing +group: group::geo default_enabled: false diff --git a/ee/app/models/ee/ci/job_artifact.rb b/ee/app/models/ee/ci/job_artifact.rb index 2cc0700e69abe1..f39ba5767e7388 100644 --- a/ee/app/models/ee/ci/job_artifact.rb +++ b/ee/app/models/ee/ci/job_artifact.rb @@ -23,7 +23,6 @@ module Ci::JobArtifact API_FUZZING_REPORT_TYPES = %w[api_fuzzing].freeze BROWSER_PERFORMANCE_REPORT_FILE_TYPES = %w[browser_performance performance].freeze - scope :project_id_in, ->(ids) { where(project_id: ids) } scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :security_reports, -> (file_types: SECURITY_REPORT_FILE_TYPES) do diff --git a/ee/app/models/ee/ci/pipeline_artifact.rb b/ee/app/models/ee/ci/pipeline_artifact.rb index 835003452b8c98..e39eaca7b8e2b2 100644 --- a/ee/app/models/ee/ci/pipeline_artifact.rb +++ b/ee/app/models/ee/ci/pipeline_artifact.rb @@ -6,19 +6,9 @@ module PipelineArtifact extend ActiveSupport::Concern prepended do - after_destroy :log_geo_deleted_event - include ::Gitlab::Geo::ReplicableModel with_replicator ::Geo::PipelineArtifactReplicator - - def self.replicables_for_current_secondary(primary_key_in) - node = ::Gitlab::Geo.current_node - - primary_key_in(primary_key_in) - .merge(selective_sync_scope(node)) - .merge(object_storage_scope(node)) - end end class_methods do @@ -46,7 +36,7 @@ def selective_sync_scope(node) end def log_geo_deleted_event - ::Geo::JobArtifactDeletedEventStore.new(self).create! + # TODO end end end diff --git a/ee/app/models/ee/terraform/state_version.rb b/ee/app/models/ee/terraform/state_version.rb index 04572af4edb214..31be62bc8980d0 100644 --- a/ee/app/models/ee/terraform/state_version.rb +++ b/ee/app/models/ee/terraform/state_version.rb @@ -7,6 +7,7 @@ module StateVersion prepended do include ::Gitlab::Geo::ReplicableModel + with_replicator Geo::TerraformStateVersionReplicator scope :project_id_in, ->(ids) { joins(:terraform_state).where('terraform_states.project_id': ids) } @@ -37,11 +38,6 @@ def selective_sync_scope(node) project_id_in(node.projects) end end - - def log_geo_deleted_event - # Keep empty for now. Should be addressed in future - # by https://gitlab.com/gitlab-org/gitlab/-/issues/232917 - end end end end diff --git a/ee/app/models/geo/pipeline_artifact_registry.rb b/ee/app/models/geo/pipeline_artifact_registry.rb index 0b967bf4889074..e1c617ec9bc5a5 100644 --- a/ee/app/models/geo/pipeline_artifact_registry.rb +++ b/ee/app/models/geo/pipeline_artifact_registry.rb @@ -3,8 +3,6 @@ module Geo class PipelineArtifactRegistry < Geo::BaseRegistry include Geo::ReplicableRegistry - include ::Gitlab::Geo::VerificationState - include ::Geo::VerifiableRegistry MODEL_CLASS = ::Ci::PipelineArtifact MODEL_FOREIGN_KEY = :pipeline_artifact_id diff --git a/ee/app/replicators/geo/pipeline_artifact_replicator.rb b/ee/app/replicators/geo/pipeline_artifact_replicator.rb index 78c99fe49cd621..2d49cfafc4014d 100644 --- a/ee/app/replicators/geo/pipeline_artifact_replicator.rb +++ b/ee/app/replicators/geo/pipeline_artifact_replicator.rb @@ -3,17 +3,19 @@ module Geo class PipelineArtifactReplicator < Gitlab::Geo::Replicator include ::Geo::BlobReplicatorStrategy + extend ::Gitlab::Utils::Override def self.model ::Ci::PipelineArtifact end - def self.replication_enabled_by_default? - false - end - def carrierwave_uploader model_record.file end + + override :verification_feature_flag_enabled? + def self.verification_feature_flag_enabled? + Feature.enabled?(:geo_pipeline_artifact_replication, default_enabled: :yaml) + end end end -- GitLab From eb4b3aacfde86a1c9ad157c703a47566494ce322 Mon Sep 17 00:00:00 2001 From: Maxime Orefice Date: Tue, 23 Mar 2021 16:48:30 -0400 Subject: [PATCH 5/6] Add verifiable logic for pipeline artifacts --- ...verification_state_to_pipeline_artifact.rb | 18 +++++++++++++ ...tion_failure_limit_to_pipeline_artifact.rb | 19 +++++++++++++ ...ification_indexes_to_pipeline_artifacts.rb | 27 +++++++++++++++++++ db/schema_migrations/20210323202729 | 1 + db/schema_migrations/20210323203055 | 1 + db/schema_migrations/20210323203227 | 1 + db/structure.sql | 18 ++++++++++++- ee/app/models/ee/ci/pipeline_artifact.rb | 5 +--- .../models/geo/pipeline_artifact_registry.rb | 3 ++- .../geo/pipeline_artifact_replicator.rb | 4 +++ 10 files changed, 91 insertions(+), 6 deletions(-) create mode 100644 db/migrate/20210323202729_add_verification_state_to_pipeline_artifact.rb create mode 100644 db/migrate/20210323203055_add_verification_failure_limit_to_pipeline_artifact.rb create mode 100644 db/migrate/20210323203227_add_verification_indexes_to_pipeline_artifacts.rb create mode 100644 db/schema_migrations/20210323202729 create mode 100644 db/schema_migrations/20210323203055 create mode 100644 db/schema_migrations/20210323203227 diff --git a/db/migrate/20210323202729_add_verification_state_to_pipeline_artifact.rb b/db/migrate/20210323202729_add_verification_state_to_pipeline_artifact.rb new file mode 100644 index 00000000000000..74bbd8ffedbb07 --- /dev/null +++ b/db/migrate/20210323202729_add_verification_state_to_pipeline_artifact.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class AddVerificationStateToPipelineArtifact < ActiveRecord::Migration[6.0] + DOWNTIME = false + + def change + change_table(:ci_pipeline_artifacts) do |t| + t.integer :verification_state, default: 0, limit: 2, null: false + t.integer :verification_retry_count, limit: 2 + t.column :verification_started_at, :datetime_with_timezone + t.column :verification_retry_at, :datetime_with_timezone + t.column :verified_at, :datetime_with_timezone + t.binary :verification_checksum, using: 'verification_checksum::bytea' + + t.text :verification_failure # rubocop:disable Migration/AddLimitToTextColumns + end + end +end diff --git a/db/migrate/20210323203055_add_verification_failure_limit_to_pipeline_artifact.rb b/db/migrate/20210323203055_add_verification_failure_limit_to_pipeline_artifact.rb new file mode 100644 index 00000000000000..77c87743e7e5ff --- /dev/null +++ b/db/migrate/20210323203055_add_verification_failure_limit_to_pipeline_artifact.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +class AddVerificationFailureLimitToPipelineArtifact < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + CONSTRAINT_NAME = 'ci_pipeline_artifacts_verification_failure_text_limit' + + def up + add_text_limit :ci_pipeline_artifacts, :verification_failure, 255, constraint_name: CONSTRAINT_NAME + end + + def down + remove_check_constraint(:ci_pipeline_artifacts, CONSTRAINT_NAME) + end +end diff --git a/db/migrate/20210323203227_add_verification_indexes_to_pipeline_artifacts.rb b/db/migrate/20210323203227_add_verification_indexes_to_pipeline_artifacts.rb new file mode 100644 index 00000000000000..fbda76c860e096 --- /dev/null +++ b/db/migrate/20210323203227_add_verification_indexes_to_pipeline_artifacts.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +class AddVerificationIndexesToPipelineArtifacts < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + VERIFICATION_STATE_INDEX_NAME = "index_ci_pipeline_artifacts_verification_state" + PENDING_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_pending_verification" + FAILED_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "index_ci_pipeline_artifacts_needs_verification" + + disable_ddl_transaction! + + def up + add_concurrent_index :ci_pipeline_artifacts, :verification_state, name: VERIFICATION_STATE_INDEX_NAME + add_concurrent_index :ci_pipeline_artifacts, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME + add_concurrent_index :ci_pipeline_artifacts, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME + add_concurrent_index :ci_pipeline_artifacts, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :ci_pipeline_artifacts, VERIFICATION_STATE_INDEX_NAME + remove_concurrent_index_by_name :ci_pipeline_artifacts, PENDING_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :ci_pipeline_artifacts, FAILED_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :ci_pipeline_artifacts, NEEDS_VERIFICATION_INDEX_NAME + end +end diff --git a/db/schema_migrations/20210323202729 b/db/schema_migrations/20210323202729 new file mode 100644 index 00000000000000..4ceab523949d63 --- /dev/null +++ b/db/schema_migrations/20210323202729 @@ -0,0 +1 @@ +3866cb66b2ab22f326f8a542538c58a0410cdb65fbefb91f5c15863a171fb4fe \ No newline at end of file diff --git a/db/schema_migrations/20210323203055 b/db/schema_migrations/20210323203055 new file mode 100644 index 00000000000000..0ce0ca10b86797 --- /dev/null +++ b/db/schema_migrations/20210323203055 @@ -0,0 +1 @@ +483153eea0df2bfd850413119ebf818a478afc821e70a6d5cb8eb1bc5bc4683a \ No newline at end of file diff --git a/db/schema_migrations/20210323203227 b/db/schema_migrations/20210323203227 new file mode 100644 index 00000000000000..94a7a245eb18ee --- /dev/null +++ b/db/schema_migrations/20210323203227 @@ -0,0 +1 @@ +b7c3bc4b39ca40234f3c675ba3f2a6100e2d2389b0e85e3eb02ac66bff64582d \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index a51e47a9317c0e..a74291934484a5 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -10672,8 +10672,16 @@ CREATE TABLE ci_pipeline_artifacts ( file_format smallint NOT NULL, file text, expire_at timestamp with time zone, + verification_state smallint DEFAULT 0 NOT NULL, + verification_retry_count smallint, + verification_started_at timestamp with time zone, + verification_retry_at timestamp with time zone, + verified_at timestamp with time zone, + verification_checksum bytea, + verification_failure text, CONSTRAINT check_191b5850ec CHECK ((char_length(file) <= 255)), - CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL)) + CONSTRAINT check_abeeb71caf CHECK ((file IS NOT NULL)), + CONSTRAINT ci_pipeline_artifacts_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255)) ); CREATE SEQUENCE ci_pipeline_artifacts_id_seq @@ -22108,6 +22116,10 @@ CREATE UNIQUE INDEX index_ci_job_variables_on_key_and_job_id ON ci_job_variables CREATE UNIQUE INDEX index_ci_namespace_monthly_usages_on_namespace_id_and_date ON ci_namespace_monthly_usages USING btree (namespace_id, date); +CREATE INDEX index_ci_pipeline_artifacts_failed_verification ON ci_pipeline_artifacts USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3); + +CREATE INDEX index_ci_pipeline_artifacts_needs_verification ON ci_pipeline_artifacts USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3)); + CREATE INDEX index_ci_pipeline_artifacts_on_expire_at ON ci_pipeline_artifacts USING btree (expire_at); CREATE INDEX index_ci_pipeline_artifacts_on_pipeline_id ON ci_pipeline_artifacts USING btree (pipeline_id); @@ -22116,6 +22128,10 @@ CREATE UNIQUE INDEX index_ci_pipeline_artifacts_on_pipeline_id_and_file_type ON CREATE INDEX index_ci_pipeline_artifacts_on_project_id ON ci_pipeline_artifacts USING btree (project_id); +CREATE INDEX index_ci_pipeline_artifacts_pending_verification ON ci_pipeline_artifacts USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0); + +CREATE INDEX index_ci_pipeline_artifacts_verification_state ON ci_pipeline_artifacts USING btree (verification_state); + CREATE INDEX index_ci_pipeline_chat_data_on_chat_name_id ON ci_pipeline_chat_data USING btree (chat_name_id); CREATE UNIQUE INDEX index_ci_pipeline_chat_data_on_pipeline_id ON ci_pipeline_chat_data USING btree (pipeline_id); diff --git a/ee/app/models/ee/ci/pipeline_artifact.rb b/ee/app/models/ee/ci/pipeline_artifact.rb index e39eaca7b8e2b2..b87dbd0fb91210 100644 --- a/ee/app/models/ee/ci/pipeline_artifact.rb +++ b/ee/app/models/ee/ci/pipeline_artifact.rb @@ -7,6 +7,7 @@ module PipelineArtifact prepended do include ::Gitlab::Geo::ReplicableModel + include ::Gitlab::Geo::VerificationState with_replicator ::Geo::PipelineArtifactReplicator end @@ -34,10 +35,6 @@ def selective_sync_scope(node) project_id_in(node.projects) end end - - def log_geo_deleted_event - # TODO - end end end end diff --git a/ee/app/models/geo/pipeline_artifact_registry.rb b/ee/app/models/geo/pipeline_artifact_registry.rb index e1c617ec9bc5a5..96602358189038 100644 --- a/ee/app/models/geo/pipeline_artifact_registry.rb +++ b/ee/app/models/geo/pipeline_artifact_registry.rb @@ -2,7 +2,8 @@ module Geo class PipelineArtifactRegistry < Geo::BaseRegistry - include Geo::ReplicableRegistry + include ::Geo::ReplicableRegistry + include ::Geo::VerifiableRegistry MODEL_CLASS = ::Ci::PipelineArtifact MODEL_FOREIGN_KEY = :pipeline_artifact_id diff --git a/ee/app/replicators/geo/pipeline_artifact_replicator.rb b/ee/app/replicators/geo/pipeline_artifact_replicator.rb index 2d49cfafc4014d..fe58f3dc84a596 100644 --- a/ee/app/replicators/geo/pipeline_artifact_replicator.rb +++ b/ee/app/replicators/geo/pipeline_artifact_replicator.rb @@ -13,6 +13,10 @@ def carrierwave_uploader model_record.file end + def self.replication_enabled_by_default? + false + end + override :verification_feature_flag_enabled? def self.verification_feature_flag_enabled? Feature.enabled?(:geo_pipeline_artifact_replication, default_enabled: :yaml) -- GitLab From 294a2058884cc5602d8d36e17e3b6f86bd89865c Mon Sep 17 00:00:00 2001 From: Maxime Orefice Date: Wed, 24 Mar 2021 12:24:14 -0400 Subject: [PATCH 6/6] Update geo node api doc --- doc/api/geo_nodes.md | 29 +++++++++++++++++++-- ee/app/models/ee/terraform/state_version.rb | 5 ++++ 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/doc/api/geo_nodes.md b/doc/api/geo_nodes.md index 0401680b016f8c..dbd0c2c918193a 100644 --- a/doc/api/geo_nodes.md +++ b/doc/api/geo_nodes.md @@ -388,7 +388,20 @@ Example response: "group_wiki_repositories_checksum_failed_count": 0, "group_wiki_repositories_registry_count": 10, "group_wiki_repositories_synced_count": 6, - "group_wiki_repositories_failed_count": 3 + "group_wiki_repositories_failed_count": 3, + "pipeline_artifacts_count": 5, + "pipeline_artifacts_checksum_total_count": 5, + "pipeline_artifacts_checksummed_count": 5, + "pipeline_artifacts_checksum_failed_count": 0, + "pipeline_artifacts_synced_count": null, + "pipeline_artifacts_failed_count": null, + "pipeline_artifacts_registry_count": null, + "pipeline_artifacts_verification_total_count": null, + "pipeline_artifacts_verified_count": null, + "pipeline_artifacts_verification_failed_count": null, + "pipeline_artifacts_synced_in_percentage": "0.00%", + "pipeline_artifacts_verified_in_percentage": "0.00%", + }, { "geo_node_id": 2, @@ -487,7 +500,19 @@ Example response: "group_wiki_repositories_checksum_failed_count": 0, "group_wiki_repositories_registry_count": 10, "group_wiki_repositories_synced_count": 6, - "group_wiki_repositories_failed_count": 3 + "group_wiki_repositories_failed_count": 3, + "pipeline_artifacts_count": 5, + "pipeline_artifacts_checksum_total_count": 5, + "pipeline_artifacts_checksummed_count": 5, + "pipeline_artifacts_checksum_failed_count": 0, + "pipeline_artifacts_synced_count": null, + "pipeline_artifacts_failed_count": null, + "pipeline_artifacts_registry_count": null, + "pipeline_artifacts_verification_total_count": null, + "pipeline_artifacts_verified_count": null, + "pipeline_artifacts_verification_failed_count": null, + "pipeline_artifacts_synced_in_percentage": "0.00%", + "pipeline_artifacts_verified_in_percentage": "0.00%", } ] ``` diff --git a/ee/app/models/ee/terraform/state_version.rb b/ee/app/models/ee/terraform/state_version.rb index 31be62bc8980d0..22adf92ce9ccc5 100644 --- a/ee/app/models/ee/terraform/state_version.rb +++ b/ee/app/models/ee/terraform/state_version.rb @@ -37,6 +37,11 @@ def selective_sync_scope(node) project_id_in(node.projects) end + + def log_geo_deleted_event + # Keep empty for now. Should be addressed in future + # by https://gitlab.com/gitlab-org/gitlab/-/issues/232917 + end end end end -- GitLab