From 2e3441502438783c649da52f2126439b49efca3c Mon Sep 17 00:00:00 2001 From: Aakriti Gupta Date: Wed, 31 Mar 2021 11:22:40 +0200 Subject: [PATCH] Geo: Add verification for terraform states - Add verification through self-service framework for Terraform States, using terraform_state_versions table - Add related metrics to API and Prometheus - Update metrics doc Changelog: changed --- app/models/terraform/state.rb | 2 ++ .../unreleased/ag-verify-terraform-state.yml | 5 +++ ...ted_at_to_terraform_state_version_table.rb | 10 ++++++ ...ion_indexes_to_terraform_state_versions.rb | 26 +++++++++++++++ db/schema_migrations/20210407140539 | 1 + db/schema_migrations/20210420173030 | 1 + db/structure.sql | 10 ++++++ .../monitoring/prometheus/gitlab_metrics.md | 6 +++- ee/app/models/ee/terraform/state.rb | 9 ++++++ ee/app/models/ee/terraform/state_version.rb | 2 ++ .../geo/terraform_state_version_registry.rb | 1 + .../geo/terraform_state_version_replicator.rb | 6 ++++ ...o_terraform_state_version_verification.yml | 8 +++++ ...ion_to_terraform_state_version_registry.rb | 15 +++++++++ ...xes_to_terraform_state_version_registry.rb | 32 +++++++++++++++++++ ee/db/geo/schema.rb | 24 +++++++++++++- ee/spec/factories/terraform/state_version.rb | 17 ++++++++++ .../terraform_state_version_registry_spec.rb | 1 + ...terraform_state_version_replicator_spec.rb | 3 +- 19 files changed, 176 insertions(+), 3 deletions(-) create mode 100644 changelogs/unreleased/ag-verify-terraform-state.yml create mode 100644 db/migrate/20210407140539_add_verification_state_and_started_at_to_terraform_state_version_table.rb create mode 100644 db/migrate/20210420173030_add_verification_indexes_to_terraform_state_versions.rb create mode 100644 db/schema_migrations/20210407140539 create mode 100644 db/schema_migrations/20210420173030 create mode 100644 ee/app/models/ee/terraform/state.rb create mode 100644 ee/config/feature_flags/development/geo_terraform_state_version_verification.yml create mode 100644 ee/db/geo/migrate/20210407140527_add_verification_to_terraform_state_version_registry.rb create mode 100644 ee/db/geo/migrate/20210420180119_add_indexes_to_terraform_state_version_registry.rb create mode 100644 ee/spec/factories/terraform/state_version.rb diff --git a/app/models/terraform/state.rb b/app/models/terraform/state.rb index eb7d465d585424..88dde62d4f2b16 100644 --- a/app/models/terraform/state.rb +++ b/app/models/terraform/state.rb @@ -104,3 +104,5 @@ def parse_serial(file) end end end + +Terraform::State.prepend_if_ee('EE::Terraform::State') diff --git a/changelogs/unreleased/ag-verify-terraform-state.yml b/changelogs/unreleased/ag-verify-terraform-state.yml new file mode 100644 index 00000000000000..6947c81884cbf1 --- /dev/null +++ b/changelogs/unreleased/ag-verify-terraform-state.yml @@ -0,0 +1,5 @@ +--- +title: 'Geo: Add verification for Terraform States' +merge_request: 58800 +author: +type: changed diff --git a/db/migrate/20210407140539_add_verification_state_and_started_at_to_terraform_state_version_table.rb b/db/migrate/20210407140539_add_verification_state_and_started_at_to_terraform_state_version_table.rb new file mode 100644 index 00000000000000..987be4ab1f0a21 --- /dev/null +++ b/db/migrate/20210407140539_add_verification_state_and_started_at_to_terraform_state_version_table.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +class AddVerificationStateAndStartedAtToTerraformStateVersionTable < ActiveRecord::Migration[6.0] + def change + change_table(:terraform_state_versions) do |t| + t.column :verification_started_at, :datetime_with_timezone + t.integer :verification_state, default: 0, limit: 2, null: false + end + end +end diff --git a/db/migrate/20210420173030_add_verification_indexes_to_terraform_state_versions.rb b/db/migrate/20210420173030_add_verification_indexes_to_terraform_state_versions.rb new file mode 100644 index 00000000000000..2c0d0bee39ddc3 --- /dev/null +++ b/db/migrate/20210420173030_add_verification_indexes_to_terraform_state_versions.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true + +class AddVerificationIndexesToTerraformStateVersions < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + VERIFICATION_STATE_INDEX_NAME = "index_terraform_state_versions_on_verification_state" + PENDING_VERIFICATION_INDEX_NAME = "index_terraform_state_versions_pending_verification" + FAILED_VERIFICATION_INDEX_NAME = "index_terraform_state_versions_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "index_terraform_state_versions_needs_verification" + + disable_ddl_transaction! + + def up + add_concurrent_index :terraform_state_versions, :verification_state, name: VERIFICATION_STATE_INDEX_NAME + add_concurrent_index :terraform_state_versions, :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME + add_concurrent_index :terraform_state_versions, :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME + add_concurrent_index :terraform_state_versions, :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME + end + + def down + remove_concurrent_index_by_name :terraform_state_versions, VERIFICATION_STATE_INDEX_NAME + remove_concurrent_index_by_name :terraform_state_versions, PENDING_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :terraform_state_versions, FAILED_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name :terraform_state_versions, NEEDS_VERIFICATION_INDEX_NAME + end +end diff --git a/db/schema_migrations/20210407140539 b/db/schema_migrations/20210407140539 new file mode 100644 index 00000000000000..3d861cfee82be9 --- /dev/null +++ b/db/schema_migrations/20210407140539 @@ -0,0 +1 @@ +9f19b44a4ef3131e6ddd9cfea0d8b1eb4499754f2200bea90b5ed41eb688f622 \ No newline at end of file diff --git a/db/schema_migrations/20210420173030 b/db/schema_migrations/20210420173030 new file mode 100644 index 00000000000000..e7e3caf836577e --- /dev/null +++ b/db/schema_migrations/20210420173030 @@ -0,0 +1 @@ +3a223c462b10edb9eb68fc0adf42f046a45f554f35b4b4ee64a834cd7372f827 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 624e74bdd9a1ee..cf331a7f4d7aa9 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -18069,6 +18069,8 @@ CREATE TABLE terraform_state_versions ( verification_checksum bytea, verification_failure text, ci_build_id bigint, + verification_started_at timestamp with time zone, + verification_state smallint DEFAULT 0 NOT NULL, CONSTRAINT check_0824bb7bbd CHECK ((char_length(file) <= 255)), CONSTRAINT tf_state_versions_verification_failure_text_limit CHECK ((char_length(verification_failure) <= 255)) ); @@ -24279,12 +24281,20 @@ CREATE INDEX index_term_agreements_on_term_id ON term_agreements USING btree (te CREATE INDEX index_term_agreements_on_user_id ON term_agreements USING btree (user_id); +CREATE INDEX index_terraform_state_versions_failed_verification ON terraform_state_versions USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3); + +CREATE INDEX index_terraform_state_versions_needs_verification ON terraform_state_versions USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3)); + CREATE INDEX index_terraform_state_versions_on_ci_build_id ON terraform_state_versions USING btree (ci_build_id); CREATE INDEX index_terraform_state_versions_on_created_by_user_id ON terraform_state_versions USING btree (created_by_user_id); CREATE UNIQUE INDEX index_terraform_state_versions_on_state_id_and_version ON terraform_state_versions USING btree (terraform_state_id, version); +CREATE INDEX index_terraform_state_versions_on_verification_state ON terraform_state_versions USING btree (verification_state); + +CREATE INDEX index_terraform_state_versions_pending_verification ON terraform_state_versions USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0); + CREATE INDEX index_terraform_states_on_file_store ON terraform_states USING btree (file_store); CREATE INDEX index_terraform_states_on_locked_by_user_id ON terraform_states USING btree (locked_by_user_id); diff --git a/doc/administration/monitoring/prometheus/gitlab_metrics.md b/doc/administration/monitoring/prometheus/gitlab_metrics.md index 6019c606c5ab17..e8cf090b19e72c 100644 --- a/doc/administration/monitoring/prometheus/gitlab_metrics.md +++ b/doc/administration/monitoring/prometheus/gitlab_metrics.md @@ -215,11 +215,15 @@ configuration option in `gitlab.yml`. These metrics are served from the | `geo_package_files_failed` | Gauge | 13.3 | Number of syncable package files failed to sync on secondary | `url` | | `geo_package_files_registry` | Gauge | 13.3 | Number of package files in the registry | `url` | | `geo_terraform_state_versions` | Gauge | 13.5 | Number of terraform state versions on primary | `url` | -| `geo_terraform_state_versions_checksummed` | Gauge | 13.5 | Number of terraform state versions checksummed on primary | `url` | +| `geo_terraform_state_versions_checksummed` | Gauge | 13.5 | Number of terraform state versions checksummed successfully on primary | `url` | | `geo_terraform_state_versions_checksum_failed` | Gauge | 13.5 | Number of terraform state versions failed to calculate the checksum on primary | `url` | +| `geo_terraform_state_versions_checksum_total` | Gauge | 13.12 | Number of terraform state versions tried to checksum on primary | `url` | | `geo_terraform_state_versions_synced` | Gauge | 13.5 | Number of syncable terraform state versions synced on secondary | `url` | | `geo_terraform_state_versions_failed` | Gauge | 13.5 | Number of syncable terraform state versions failed to sync on secondary | `url` | | `geo_terraform_state_versions_registry` | Gauge | 13.5 | Number of terraform state versions in the registry | `url` | +| `geo_terraform_state_versions_verified` | Gauge | 13.12 | Number of terraform state versions verified on secondary | `url` | +| `geo_terraform_state_versions_verification_failed` | Gauge | 13.12 | Number of terraform state versions verifications failed on secondary | `url` | +| `geo_terraform_state_versions_verification_total` | Gauge | 13.12 | Number of terraform state versions verifications tried on secondary | `url` | | `global_search_bulk_cron_queue_size` | Gauge | 12.10 | Number of database records waiting to be synchronized to Elasticsearch | | | `global_search_awaiting_indexing_queue_size` | Gauge | 13.2 | Number of database updates waiting to be synchronized to Elasticsearch while indexing is paused | | | `geo_merge_request_diffs` | Gauge | 13.4 | Number of merge request diffs on primary | `url` | diff --git a/ee/app/models/ee/terraform/state.rb b/ee/app/models/ee/terraform/state.rb new file mode 100644 index 00000000000000..9333c1f1c32c80 --- /dev/null +++ b/ee/app/models/ee/terraform/state.rb @@ -0,0 +1,9 @@ +# frozen_string_literal: true + +module EE + module Terraform + module State + extend ActiveSupport::Concern + end + end +end diff --git a/ee/app/models/ee/terraform/state_version.rb b/ee/app/models/ee/terraform/state_version.rb index 04572af4edb214..581bb2036b74f6 100644 --- a/ee/app/models/ee/terraform/state_version.rb +++ b/ee/app/models/ee/terraform/state_version.rb @@ -7,6 +7,8 @@ module StateVersion prepended do include ::Gitlab::Geo::ReplicableModel + include ::Gitlab::Geo::VerificationState + with_replicator Geo::TerraformStateVersionReplicator scope :project_id_in, ->(ids) { joins(:terraform_state).where('terraform_states.project_id': ids) } diff --git a/ee/app/models/geo/terraform_state_version_registry.rb b/ee/app/models/geo/terraform_state_version_registry.rb index f31d687436ca6c..e9e8adb4cf491e 100644 --- a/ee/app/models/geo/terraform_state_version_registry.rb +++ b/ee/app/models/geo/terraform_state_version_registry.rb @@ -2,6 +2,7 @@ class Geo::TerraformStateVersionRegistry < Geo::BaseRegistry include Geo::ReplicableRegistry + include ::Geo::VerifiableRegistry MODEL_CLASS = ::Terraform::StateVersion MODEL_FOREIGN_KEY = :terraform_state_version_id diff --git a/ee/app/replicators/geo/terraform_state_version_replicator.rb b/ee/app/replicators/geo/terraform_state_version_replicator.rb index 30f3daaf17021f..eaca219e0fb0bb 100644 --- a/ee/app/replicators/geo/terraform_state_version_replicator.rb +++ b/ee/app/replicators/geo/terraform_state_version_replicator.rb @@ -3,6 +3,7 @@ module Geo class TerraformStateVersionReplicator < Gitlab::Geo::Replicator include ::Geo::BlobReplicatorStrategy + extend ::Gitlab::Utils::Override def carrierwave_uploader model_record.file @@ -11,5 +12,10 @@ def carrierwave_uploader def self.model ::Terraform::StateVersion end + + override :verification_feature_flag_enabled? + def self.verification_feature_flag_enabled? + Feature.enabled?(:geo_terraform_state_version_verification, default_enabled: :yaml) + end end end diff --git a/ee/config/feature_flags/development/geo_terraform_state_version_verification.yml b/ee/config/feature_flags/development/geo_terraform_state_version_verification.yml new file mode 100644 index 00000000000000..4029fbf3786835 --- /dev/null +++ b/ee/config/feature_flags/development/geo_terraform_state_version_verification.yml @@ -0,0 +1,8 @@ +--- +name: geo_terraform_state_version_verification +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/58800 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/329963 +milestone: '13.12' +type: development +group: group::geo +default_enabled: false diff --git a/ee/db/geo/migrate/20210407140527_add_verification_to_terraform_state_version_registry.rb b/ee/db/geo/migrate/20210407140527_add_verification_to_terraform_state_version_registry.rb new file mode 100644 index 00000000000000..c8fe5407e369dc --- /dev/null +++ b/ee/db/geo/migrate/20210407140527_add_verification_to_terraform_state_version_registry.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +class AddVerificationToTerraformStateVersionRegistry < ActiveRecord::Migration[6.0] + def change + add_column :terraform_state_version_registry, :verification_started_at, :datetime_with_timezone + add_column :terraform_state_version_registry, :verified_at, :datetime_with_timezone + add_column :terraform_state_version_registry, :verification_retry_at, :datetime_with_timezone + add_column :terraform_state_version_registry, :verification_retry_count, :integer, default: 0 + add_column :terraform_state_version_registry, :verification_state, :integer, limit: 2, default: 0, null: false + add_column :terraform_state_version_registry, :checksum_mismatch, :boolean, default: false, null: false + add_column :terraform_state_version_registry, :verification_checksum, :binary + add_column :terraform_state_version_registry, :verification_checksum_mismatched, :binary + add_column :terraform_state_version_registry, :verification_failure, :string, limit: 255 # rubocop:disable Migration/PreventStrings because https://gitlab.com/gitlab-org/gitlab/-/issues/323806 + end +end diff --git a/ee/db/geo/migrate/20210420180119_add_indexes_to_terraform_state_version_registry.rb b/ee/db/geo/migrate/20210420180119_add_indexes_to_terraform_state_version_registry.rb new file mode 100644 index 00000000000000..c2b769bd4c579a --- /dev/null +++ b/ee/db/geo/migrate/20210420180119_add_indexes_to_terraform_state_version_registry.rb @@ -0,0 +1,32 @@ +# frozen_string_literal: true + +class AddIndexesToTerraformStateVersionRegistry < ActiveRecord::Migration[6.0] + include Gitlab::Database::MigrationHelpers + + TERRAFORM_STATE_VERSION_ID_INDEX_NAME = "index_terraform_state_version_registry_on_t_state_version_id" + FAILED_VERIFICATION_INDEX_NAME = "terraform_state_version_registry_failed_verification" + NEEDS_VERIFICATION_INDEX_NAME = "terraform_state_version_registry_needs_verification" + PENDING_VERIFICATION_INDEX_NAME = "terraform_state_version_registry_pending_verification" + + REGISTRY_TABLE = :terraform_state_version_registry + + disable_ddl_transaction! + + def up + add_concurrent_index REGISTRY_TABLE, :terraform_state_version_id, name: TERRAFORM_STATE_VERSION_ID_INDEX_NAME, unique: true + add_concurrent_index REGISTRY_TABLE, :retry_at + add_concurrent_index REGISTRY_TABLE, :state + add_concurrent_index REGISTRY_TABLE, :verification_retry_at, name: FAILED_VERIFICATION_INDEX_NAME, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + add_concurrent_index REGISTRY_TABLE, :verification_state, name: NEEDS_VERIFICATION_INDEX_NAME, where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + add_concurrent_index REGISTRY_TABLE, :verified_at, name: PENDING_VERIFICATION_INDEX_NAME, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" + end + + def down + remove_concurrent_index_by_name REGISTRY_TABLE, name: TERRAFORM_STATE_VERSION_ID_INDEX_NAME + remove_concurrent_index_by_name REGISTRY_TABLE, name: :index_terraform_state_version_registry_on_retry_at + remove_concurrent_index_by_name REGISTRY_TABLE, name: :index_terraform_state_version_registry_on_state + remove_concurrent_index_by_name REGISTRY_TABLE, name: FAILED_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name REGISTRY_TABLE, name: NEEDS_VERIFICATION_INDEX_NAME + remove_concurrent_index_by_name REGISTRY_TABLE, name: PENDING_VERIFICATION_INDEX_NAME + end +end diff --git a/ee/db/geo/schema.rb b/ee/db/geo/schema.rb index 796560b4cad103..8f4dfe823e9b40 100644 --- a/ee/db/geo/schema.rb +++ b/ee/db/geo/schema.rb @@ -10,7 +10,7 @@ # # It's strongly recommended that you check this file into your version control system. -ActiveRecord::Schema.define(version: 2021_03_25_150435) do +ActiveRecord::Schema.define(version: 2021_04_20_180119) do # These are extensions that must be enabled in order to support this database enable_extension "plpgsql" @@ -118,6 +118,15 @@ t.integer "state", limit: 2, default: 0, null: false t.integer "retry_count", limit: 2, default: 0 t.text "last_sync_failure" + t.datetime_with_timezone "verification_started_at" + t.datetime_with_timezone "verified_at" + t.datetime_with_timezone "verification_retry_at" + t.integer "verification_retry_count" + t.integer "verification_state", limit: 2, default: 0, null: false + t.boolean "checksum_mismatch" + t.binary "verification_checksum" + t.binary "verification_checksum_mismatched" + t.string "verification_failure", limit: 255 t.index ["merge_request_diff_id"], name: "index_merge_request_diff_registry_on_mr_diff_id" t.index ["retry_at"], name: "index_merge_request_diff_registry_on_retry_at" t.index ["state"], name: "index_merge_request_diff_registry_on_state" @@ -271,9 +280,22 @@ t.datetime_with_timezone "last_synced_at" t.datetime_with_timezone "created_at", null: false t.text "last_sync_failure" + t.datetime_with_timezone "verification_started_at" + t.datetime_with_timezone "verified_at" + t.datetime_with_timezone "verification_retry_at" + t.integer "verification_retry_count", default: 0 + t.integer "verification_state", limit: 2, default: 0, null: false + t.boolean "checksum_mismatch", default: false, null: false + t.binary "verification_checksum" + t.binary "verification_checksum_mismatched" + t.string "verification_failure", limit: 255 t.index ["retry_at"], name: "index_terraform_state_version_registry_on_retry_at" t.index ["state"], name: "index_terraform_state_version_registry_on_state" + t.index ["terraform_state_version_id"], name: "index_terraform_state_version_registry_on_t_state_version_id", unique: true t.index ["terraform_state_version_id"], name: "index_tf_state_versions_registry_tf_state_versions_id_unique", unique: true + t.index ["verification_retry_at"], name: "terraform_state_version_registry_failed_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))" + t.index ["verification_state"], name: "terraform_state_version_registry_needs_verification", where: "((state = 2) AND (verification_state = ANY (ARRAY[0, 3])))" + t.index ["verified_at"], name: "terraform_state_version_registry_pending_verification", order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))" end end diff --git a/ee/spec/factories/terraform/state_version.rb b/ee/spec/factories/terraform/state_version.rb new file mode 100644 index 00000000000000..53c5c69c3b4894 --- /dev/null +++ b/ee/spec/factories/terraform/state_version.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +FactoryBot.modify do + factory :terraform_state_version, class: '::Terraform::StateVersion' do + trait(:verification_succeeded) do + with_file + verification_checksum { 'abc' } + verification_state { ::Terraform::StateVersion.verification_state_value(:verification_succeeded) } + end + + trait(:verification_failed) do + with_file + verification_failure { 'Could not calculate the checksum' } + verification_state { ::Terraform::StateVersion.verification_state_value(:verification_failed) } + end + end +end diff --git a/ee/spec/models/geo/terraform_state_version_registry_spec.rb b/ee/spec/models/geo/terraform_state_version_registry_spec.rb index 1e9af786d964dc..c923a76c98cce0 100644 --- a/ee/spec/models/geo/terraform_state_version_registry_spec.rb +++ b/ee/spec/models/geo/terraform_state_version_registry_spec.rb @@ -10,4 +10,5 @@ end include_examples 'a Geo framework registry' + include_examples 'a Geo verifiable registry' end diff --git a/ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb b/ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb index 3281deb7398502..a8973a520f274d 100644 --- a/ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb +++ b/ee/spec/replicators/geo/terraform_state_version_replicator_spec.rb @@ -5,5 +5,6 @@ RSpec.describe Geo::TerraformStateVersionReplicator do let(:model_record) { build(:terraform_state_version, terraform_state: create(:terraform_state)) } - it_behaves_like 'a blob replicator' + include_examples 'a blob replicator' + include_examples 'a verifiable replicator' end -- GitLab