diff --git a/app/models/terraform/state.rb b/app/models/terraform/state.rb index eb7d465d5854244c62f225eb381e90abcfd2d130..88dde62d4f2b16baea050559d8b24a629ecd56f8 100644 --- a/app/models/terraform/state.rb +++ b/app/models/terraform/state.rb @@ -104,3 +104,5 @@ def parse_serial(file) end end end + +Terraform::State.prepend_if_ee('EE::Terraform::State') diff --git a/changelogs/unreleased/ag-verify-terraform-state.yml b/changelogs/unreleased/ag-verify-terraform-state.yml new file mode 100644 index 0000000000000000000000000000000000000000..6947c81884cbf1d082ea4bc828a595757440f971 --- /dev/null +++ b/changelogs/unreleased/ag-verify-terraform-state.yml @@ -0,0 +1,5 @@ +--- +title: 'Geo: Add verification for Terraform States' +merge_request: 58800 +author: +type: changed diff --git a/db/migrate/20210407140539_add_verification_state_and_started_at_to_terraform_state_version_table.rb b/db/migrate/20210407140539_add_verification_state_and_started_at_to_terraform_state_version_table.rb new file mode 100644 index 0000000000000000000000000000000000000000..987be4ab1f0a21f6c1a0f52131bdcb334be4f86c --- /dev/null +++ b/db/migrate/20210407140539_add_verification_state_and_started_at_to_terraform_state_version_table.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class AddVerificationStateAndStartedAtToTerraformStateVersionTable < ActiveRecord::Migration[6.0] + def change + change_table(:terraform_state_versions) do |t| + t.column :verification_started_at, :datetime_with_timezone + t.integer :verification_state, default: 0, limit: 2, null: false + end + end +end diff --git a/db/migrate/20210420173030_add_verification_indexes_to_terraform_state_versions.rb b/db/migrate/20210420173030_add_verification_indexes_to_terraform_state_versions.rb new file mode 100644 index 0000000000000000000000000000000000000000..2c0d0bee39ddc3fed9be2b6b6352a172864b231d --- /dev/null +++ b/db/migrate/20210420173030_add_verification_indexes_to_terraform_state_versions.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class AddVerificationIndexesToTerraformStateVersions < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + VERIFICATION_STATE_INDEX_NAME = "index_terraform_state_versions_on_verification_state" + PENDING_VERIFICATION_INDEX_NAME = "index_terraform_state_versions_pending_verification" + FAILED_VERIFICATION_INDEX_NAME = "index_terraform_state_versions_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "index_terraform_state_versions_needs_verification" + + disable_ddl_transaction! + + def up + add_concurrent_index :terraform_state_versions, :verification_state, name: VERIFICATION_STATE_INDEX_NAME + add_concurrent_index :terraform_state_versions, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME + add_concurrent_index :terraform_state_versions, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME + add_concurrent_index :terraform_state_versions, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :terraform_state_versions, VERIFICATION_STATE_INDEX_NAME + remove_concurrent_index_by_name :terraform_state_versions, PENDING_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :terraform_state_versions, FAILED_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :terraform_state_versions, NEEDS_VERIFICATION_INDEX_NAME + end +end diff --git a/db/schema_migrations/20210407140539 b/db/schema_migrations/20210407140539 new file mode 100644 index 0000000000000000000000000000000000000000..3d861cfee82be9f9e6b3577786180e314a195e7e --- /dev/null +++ b/db/schema_migrations/20210407140539 @@ -0,0 +1 @@ +9f19b44a4ef3131e6ddd9cfea0d8b1eb4499754f2200bea90b5ed41eb688f622 \ No newline at end of file diff --git a/db/schema_migrations/20210420173030 b/db/schema_migrations/20210420173030 new file mode 100644 index 0000000000000000000000000000000000000000..e7e3caf836577e1d44820760f431a70882b202f2 --- /dev/null +++ b/db/schema_migrations/20210420173030 @@ -0,0 +1 @@ +3a223c462b10edb9eb68fc0adf42f046a45f554f35b4b4ee64a834cd7372f827 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 624e74bdd9a1eeb95b9c5c2b4de257ce0448eee0..cf331a7f4d7aa9207d64f55fe6507348c1d98c2d 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -18069,6 +18069,8 @@ CREATE TABLE terraform_state_versions ( verification_checksum bytea, verification_failure text, ci_build_id bigint, + verification_started_at timestamp with time zone, + verification_state smallint DEFAULT 0 NOT NULL, CONSTRAINT check_0824bb7bbd CHECK ((char_length(file) <= 255)), CONSTRAINT tf_state_versions_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255)) ); @@ -24279,12 +24281,20 @@ CREATE INDEX index_term_agreements_on_term_id ON term_agreements USING btree (te CREATE INDEX index_term_agreements_on_user_id ON term_agreements USING btree (user_id); +CREATE INDEX index_terraform_state_versions_failed_verification ON terraform_state_versions USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3); + +CREATE INDEX index_terraform_state_versions_needs_verification ON terraform_state_versions USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3)); + CREATE INDEX index_terraform_state_versions_on_ci_build_id ON terraform_state_versions USING btree (ci_build_id); CREATE INDEX index_terraform_state_versions_on_created_by_user_id ON terraform_state_versions USING btree (created_by_user_id); CREATE UNIQUE INDEX index_terraform_state_versions_on_state_id_and_version ON terraform_state_versions USING btree (terraform_state_id, version); +CREATE INDEX index_terraform_state_versions_on_verification_state ON terraform_state_versions USING btree (verification_state); + +CREATE INDEX index_terraform_state_versions_pending_verification ON terraform_state_versions USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0); + CREATE INDEX index_terraform_states_on_file_store ON terraform_states USING btree (file_store); CREATE INDEX index_terraform_states_on_locked_by_user_id ON terraform_states USING btree (locked_by_user_id); diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 6019c606c5ab171a8cfc1d2f8b523b2c9ebdc1a8..e8cf090b19e72cea127f95c0addf149972fb9cf8 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -215,11 +215,15 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_package_files_failed` | Gauge | 13.3 | Number of syncable package files failed to sync on secondary | `url` | | `geo_package_files_registry` | Gauge | 13.3 | Number of package files in the registry | `url` | | `geo_terraform_state_versions` | Gauge | 13.5 | Number of terraform state versions on primary | `url` | -| `geo_terraform_state_versions_checksummed` | Gauge | 13.5 | Number of terraform state versions checksummed on primary | `url` | +| `geo_terraform_state_versions_checksummed` | Gauge | 13.5 | Number of terraform state versions checksummed successfully on primary | `url` | | `geo_terraform_state_versions_checksum_failed` | Gauge | 13.5 | Number of terraform state versions failed to calculate the checksum on primary | `url` | +| `geo_terraform_state_versions_checksum_total` | Gauge | 13.12 | Number of terraform state versions tried to checksum on primary | `url` | | `geo_terraform_state_versions_synced` | Gauge | 13.5 | Number of syncable terraform state versions synced on secondary | `url` | | `geo_terraform_state_versions_failed` | Gauge | 13.5 | Number of syncable terraform state versions failed to sync on secondary | `url` | | `geo_terraform_state_versions_registry` | Gauge | 13.5 | Number of terraform state versions in the registry | `url` | +| `geo_terraform_state_versions_verified` | Gauge | 13.12 | Number of terraform state versions verified on secondary | `url` | +| `geo_terraform_state_versions_verification_failed` | Gauge | 13.12 | Number of terraform state versions verifications failed on secondary | `url` | +| `geo_terraform_state_versions_verification_total` | Gauge | 13.12 | Number of terraform state versions verifications tried on secondary | `url` | | `global_search_bulk_cron_queue_size` | Gauge | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | | | `global_search_awaiting_indexing_queue_size` | Gauge | 13.2 | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | | | `geo_merge_request_diffs` | Gauge | 13.4 | Number of merge request diffs on primary | `url` | diff --git a/ee/app/models/ee/terraform/state.rb b/ee/app/models/ee/terraform/state.rb new file mode 100644 index 0000000000000000000000000000000000000000..9333c1f1c32c803de5553eade0782b76f61945e8 --- /dev/null +++ b/ee/app/models/ee/terraform/state.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module EE + module Terraform + module State + extend ActiveSupport::Concern + end + end +end diff --git a/ee/app/models/ee/terraform/state_version.rb b/ee/app/models/ee/terraform/state_version.rb index 04572af4edb2142bc7cc58c2561275f0237e287b..581bb2036b74f6830430d52b34908b7fa46a6882 100644 --- a/ee/app/models/ee/terraform/state_version.rb +++ b/ee/app/models/ee/terraform/state_version.rb @@ -7,6 +7,8 @@ module StateVersion prepended do include ::Gitlab::Geo::ReplicableModel + include ::Gitlab::Geo::VerificationState + with_replicator Geo::TerraformStateVersionReplicator scope :project_id_in, ->(ids) { joins(:terraform_state).where('terraform_states.project_id': ids) } diff --git a/ee/app/models/geo/terraform_state_version_registry.rb b/ee/app/models/geo/terraform_state_version_registry.rb index f31d687436ca6cd86a035813e6b613c48b55fd2b..e9e8adb4cf491edc7b3db6254ac746ec2e4c0a11 100644 --- a/ee/app/models/geo/terraform_state_version_registry.rb +++ b/ee/app/models/geo/terraform_state_version_registry.rb @@ -2,6 +2,7 @@ class Geo::TerraformStateVersionRegistry < Geo::BaseRegistry include Geo::ReplicableRegistry + include ::Geo::VerifiableRegistry MODEL_CLASS = ::Terraform::StateVersion MODEL_FOREIGN_KEY = :terraform_state_version_id diff --git a/ee/app/replicators/geo/terraform_state_version_replicator.rb b/ee/app/replicators/geo/terraform_state_version_replicator.rb index 30f3daaf17021f6e7633bfdf5af629bfae14db25..eaca219e0fb0bbe71793e25e183f66b73eae0b6d 100644 --- a/ee/app/replicators/geo/terraform_state_version_replicator.rb +++ b/ee/app/replicators/geo/terraform_state_version_replicator.rb @@ -3,6 +3,7 @@ module Geo class TerraformStateVersionReplicator < Gitlab::Geo::Replicator include ::Geo::BlobReplicatorStrategy + extend ::Gitlab::Utils::Override def carrierwave_uploader model_record.file @@ -11,5 +12,10 @@ def carrierwave_uploader def self.model ::Terraform::StateVersion end + + override :verification_feature_flag_enabled? + def self.verification_feature_flag_enabled? + Feature.enabled?(:geo_terraform_state_version_verification, default_enabled: :yaml) + end end end diff --git a/ee/config/feature_flags/development/geo_terraform_state_version_verification.yml b/ee/config/feature_flags/development/geo_terraform_state_version_verification.yml new file mode 100644 index 0000000000000000000000000000000000000000..4029fbf3786835ddedd5f62a6ea53e869dcaf1f7 --- /dev/null +++ b/ee/config/feature_flags/development/geo_terraform_state_version_verification.yml @@ -0,0 +1,8 @@ +--- +name: geo_terraform_state_version_verification +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58800 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/329963 +milestone: '13.12' +type: development +group: group::geo +default_enabled: false diff --git a/ee/db/geo/migrate/20210407140527_add_verification_to_terraform_state_version_registry.rb b/ee/db/geo/migrate/20210407140527_add_verification_to_terraform_state_version_registry.rb new file mode 100644 index 0000000000000000000000000000000000000000..c8fe5407e369dcfc74eed2e675ff68653944a938 --- /dev/null +++ b/ee/db/geo/migrate/20210407140527_add_verification_to_terraform_state_version_registry.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddVerificationToTerraformStateVersionRegistry < ActiveRecord::Migration[6.0] + def change + add_column :terraform_state_version_registry, :verification_started_at, :datetime_with_timezone + add_column :terraform_state_version_registry, :verified_at, :datetime_with_timezone + add_column :terraform_state_version_registry, :verification_retry_at, :datetime_with_timezone + add_column :terraform_state_version_registry, :verification_retry_count, :integer, default: 0 + add_column :terraform_state_version_registry, :verification_state, :integer, limit: 2, default: 0, null: false + add_column :terraform_state_version_registry, :checksum_mismatch, :boolean, default: false, null: false + add_column :terraform_state_version_registry, :verification_checksum, :binary + add_column :terraform_state_version_registry, :verification_checksum_mismatched, :binary + add_column :terraform_state_version_registry, :verification_failure, :string, limit: 255 # rubocop:disable Migration/PreventStrings because https://gitlab.com/gitlab-org/gitlab/-/issues/323806 + end +end diff --git a/ee/db/geo/migrate/20210420180119_add_indexes_to_terraform_state_version_registry.rb b/ee/db/geo/migrate/20210420180119_add_indexes_to_terraform_state_version_registry.rb new file mode 100644 index 0000000000000000000000000000000000000000..c2b769bd4c579a9554ab5d6419e06be8a1409285 --- /dev/null +++ b/ee/db/geo/migrate/20210420180119_add_indexes_to_terraform_state_version_registry.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class AddIndexesToTerraformStateVersionRegistry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + TERRAFORM_STATE_VERSION_ID_INDEX_NAME = "index_terraform_state_version_registry_on_t_state_version_id" + FAILED_VERIFICATION_INDEX_NAME = "terraform_state_version_registry_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "terraform_state_version_registry_needs_verification" + PENDING_VERIFICATION_INDEX_NAME = "terraform_state_version_registry_pending_verification" + + REGISTRY_TABLE = :terraform_state_version_registry + + disable_ddl_transaction! + + def up + add_concurrent_index REGISTRY_TABLE, :terraform_state_version_id, name: TERRAFORM_STATE_VERSION_ID_INDEX_NAME, unique: true + add_concurrent_index REGISTRY_TABLE, :retry_at + add_concurrent_index REGISTRY_TABLE, :state + add_concurrent_index REGISTRY_TABLE, :verification_retry_at, name: FAILED_VERIFICATION_INDEX_NAME, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + add_concurrent_index REGISTRY_TABLE, :verification_state, name: NEEDS_VERIFICATION_INDEX_NAME, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + add_concurrent_index REGISTRY_TABLE, :verified_at, name: PENDING_VERIFICATION_INDEX_NAME, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" + end + + def down + remove_concurrent_index_by_name REGISTRY_TABLE, name: TERRAFORM_STATE_VERSION_ID_INDEX_NAME + remove_concurrent_index_by_name REGISTRY_TABLE, name: :index_terraform_state_version_registry_on_retry_at + remove_concurrent_index_by_name REGISTRY_TABLE, name: :index_terraform_state_version_registry_on_state + remove_concurrent_index_by_name REGISTRY_TABLE, name: FAILED_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name REGISTRY_TABLE, name: NEEDS_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name REGISTRY_TABLE, name: PENDING_VERIFICATION_INDEX_NAME + end +end diff --git a/ee/db/geo/schema.rb b/ee/db/geo/schema.rb index 796560b4cad1036fbdb2dc2b03653fd39dbb43cd..8f4dfe823e9b4068a03f9154fdc98705cfa92c5f 100644 --- a/ee/db/geo/schema.rb +++ b/ee/db/geo/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_03_25_150435) do +ActiveRecord::Schema.define(version: 2021_04_20_180119) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -118,6 +118,15 @@ t.integer "state", limit: 2, default: 0, null: false t.integer "retry_count", limit: 2, default: 0 t.text "last_sync_failure" + t.datetime_with_timezone "verification_started_at" + t.datetime_with_timezone "verified_at" + t.datetime_with_timezone "verification_retry_at" + t.integer "verification_retry_count" + t.integer "verification_state", limit: 2, default: 0, null: false + t.boolean "checksum_mismatch" + t.binary "verification_checksum" + t.binary "verification_checksum_mismatched" + t.string "verification_failure", limit: 255 t.index ["merge_request_diff_id"], name: "index_merge_request_diff_registry_on_mr_diff_id" t.index ["retry_at"], name: "index_merge_request_diff_registry_on_retry_at" t.index ["state"], name: "index_merge_request_diff_registry_on_state" @@ -271,9 +280,22 @@ t.datetime_with_timezone "last_synced_at" t.datetime_with_timezone "created_at", null: false t.text "last_sync_failure" + t.datetime_with_timezone "verification_started_at" + t.datetime_with_timezone "verified_at" + t.datetime_with_timezone "verification_retry_at" + t.integer "verification_retry_count", default: 0 + t.integer "verification_state", limit: 2, default: 0, null: false + t.boolean "checksum_mismatch", default: false, null: false + t.binary "verification_checksum" + t.binary "verification_checksum_mismatched" + t.string "verification_failure", limit: 255 t.index ["retry_at"], name: "index_terraform_state_version_registry_on_retry_at" t.index ["state"], name: "index_terraform_state_version_registry_on_state" + t.index ["terraform_state_version_id"], name: "index_terraform_state_version_registry_on_t_state_version_id", unique: true t.index ["terraform_state_version_id"], name: "index_tf_state_versions_registry_tf_state_versions_id_unique", unique: true + t.index ["verification_retry_at"], name: "terraform_state_version_registry_failed_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + t.index ["verification_state"], name: "terraform_state_version_registry_needs_verification", where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + t.index ["verified_at"], name: "terraform_state_version_registry_pending_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" end end diff --git a/ee/spec/factories/terraform/state_version.rb b/ee/spec/factories/terraform/state_version.rb new file mode 100644 index 0000000000000000000000000000000000000000..53c5c69c3b48943d9e527b3c5e523c6635dcad0a --- /dev/null +++ b/ee/spec/factories/terraform/state_version.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +FactoryBot.modify do + factory :terraform_state_version, class: '::Terraform::StateVersion' do + trait(:verification_succeeded) do + with_file + verification_checksum { 'abc' } + verification_state { ::Terraform::StateVersion.verification_state_value(:verification_succeeded) } + end + + trait(:verification_failed) do + with_file + verification_failure { 'Could not calculate the checksum' } + verification_state { ::Terraform::StateVersion.verification_state_value(:verification_failed) } + end + end +end diff --git a/ee/spec/models/geo/terraform_state_version_registry_spec.rb b/ee/spec/models/geo/terraform_state_version_registry_spec.rb index 1e9af786d964dc8beb8fe38426f9229422022f56..c923a76c98cce02844ce4700fd2ec413ca6a3748 100644 --- a/ee/spec/models/geo/terraform_state_version_registry_spec.rb +++ b/ee/spec/models/geo/terraform_state_version_registry_spec.rb @@ -10,4 +10,5 @@ end include_examples 'a Geo framework registry' + include_examples 'a Geo verifiable registry' end diff --git a/ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb b/ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb index 3281deb739850238bda70c507e6a8fddf663be8f..a8973a520f274d844357c480660928ef04d10e16 100644 --- a/ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb +++ b/ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb @@ -5,5 +5,6 @@ RSpec.describe Geo::TerraformStateVersionReplicator do let(:model_record) { build(:terraform_state_version, terraform_state: create(:terraform_state)) } - it_behaves_like 'a blob replicator' + include_examples 'a blob replicator' + include_examples 'a verifiable replicator' end