From 931877cc7bb76f674cb02faa7efe51cb60fbd8c9 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Tue, 15 Jul 2025 18:55:53 -0500 Subject: [PATCH 01/13] Create model to store metadata for SLSA provenance attestations Adds the Ci::Slsa::Attestation model and DB table. Records will refer to signature bundles which will be kept in file/object storage. Changelog: added --- app/models/ci/slsa/attestation.rb | 30 +++++++++++++++++++ db/docs/ci_slsa_attestations.yml | 13 ++++++++ ...50714052826_create_ci_slsa_attestations.rb | 21 +++++++++++++ db/schema_migrations/20250714052826 | 1 + db/structure.sql | 29 ++++++++++++++++++ spec/factories/ci/slsa/attestations.rb | 10 +++++++ spec/models/ci/slsa/attestation_spec.rb | 17 +++++++++++ 7 files changed, 121 insertions(+) create mode 100644 app/models/ci/slsa/attestation.rb create mode 100644 db/docs/ci_slsa_attestations.yml create mode 100644 db/migrate/20250714052826_create_ci_slsa_attestations.rb create mode 100644 db/schema_migrations/20250714052826 create mode 100644 spec/factories/ci/slsa/attestations.rb create mode 100644 spec/models/ci/slsa/attestation_spec.rb diff --git a/app/models/ci/slsa/attestation.rb b/app/models/ci/slsa/attestation.rb new file mode 100644 index 00000000000000..77e74315c74209 --- /dev/null +++ b/app/models/ci/slsa/attestation.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true + +module Ci + module Slsa + class Attestation < SecApplicationRecord + self.table_name = 'ci_slsa_attestations' + + belongs_to :project, optional: false + + validates :project_id, presence: true + validates :predicate_kind, presence: true + validates :predicate_type, presence: true + validates :subject_digest, presence: true, length: { minimum: 64, maximum: 255 } + + validates :subject_digest, uniqueness: { scope: [:project_id, :predicate_kind] } + + enum :status, { + success: 0, + error: 1 + } + + enum :predicate_kind, { + provenance: 0, + sbom: 1 + } + + scope :by_digest, ->(digest) { where(digest: digest) } + end + end +end diff --git a/db/docs/ci_slsa_attestations.yml b/db/docs/ci_slsa_attestations.yml new file mode 100644 index 00000000000000..bc65f7ceef99c9 --- /dev/null +++ b/db/docs/ci_slsa_attestations.yml @@ -0,0 +1,13 @@ +--- +table_name: ci_slsa_attestations +description: Stores SLSA attestations generated by the GitLab Trusted Control Plane +introduced_by_url: TBD +milestone: '18.3' +feature_categories: +- artifact_security +classes: +- Ci::Slsa::Attestation +sharding_key: + project_id: projects +gitlab_schema: gitlab_sec +table_size: small \ No newline at end of file diff --git a/db/migrate/20250714052826_create_ci_slsa_attestations.rb b/db/migrate/20250714052826_create_ci_slsa_attestations.rb new file mode 100644 index 00000000000000..394a42052b0e57 --- /dev/null +++ b/db/migrate/20250714052826_create_ci_slsa_attestations.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true + +class CreateCiSlsaAttestations < Gitlab::Database::Migration[2.3] + milestone '18.3' + + INDEX_NAME = 'idx_uniq_slsa_attestations_on_digest_project_id_predicate_kind' + + def change + create_table :ci_slsa_attestations do |t| + t.integer :status, null: false, default: 0, limit: 2 + t.text :subject_digest, null: false, limit: 255 + t.timestamps_with_timezone null: false + t.bigint :project_id, null: false + t.integer :predicate_kind, null: false, default: 0, limit: 2 + t.text :predicate_type, null: false, limit: 255 + end + + add_index :ci_slsa_attestations, [:subject_digest, :project_id, :predicate_kind], + name: INDEX_NAME, unique: true + end +end diff --git a/db/schema_migrations/20250714052826 b/db/schema_migrations/20250714052826 new file mode 100644 index 00000000000000..b353dfb5a86823 --- /dev/null +++ b/db/schema_migrations/20250714052826 @@ -0,0 +1 @@ +fc1498e1f9ffc86c2aa78dc524f70cad7fcf1f5ccf3a0ea88105374c0f157215 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 8cffc3ca96beec..5029967fbaee8b 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12526,6 +12526,28 @@ CREATE SEQUENCE ci_secure_files_id_seq ALTER SEQUENCE ci_secure_files_id_seq OWNED BY ci_secure_files.id; +CREATE TABLE ci_slsa_attestations ( + id bigint NOT NULL, + status smallint DEFAULT 0 NOT NULL, + subject_digest text NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + project_id bigint NOT NULL, + predicate_kind smallint DEFAULT 0 NOT NULL, + predicate_type text NOT NULL, + CONSTRAINT check_7966ba494c CHECK ((char_length(subject_digest) <= 255)), + CONSTRAINT check_85b36b80e0 CHECK ((char_length(predicate_type) <= 255)) +); + +CREATE SEQUENCE ci_slsa_attestations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ci_slsa_attestations_id_seq OWNED BY ci_slsa_attestations.id; + CREATE TABLE ci_sources_pipelines ( id bigint NOT NULL, project_id bigint, @@ -28182,6 +28204,8 @@ ALTER TABLE ONLY ci_secure_file_states ALTER COLUMN ci_secure_file_id SET DEFAUL ALTER TABLE ONLY ci_secure_files ALTER COLUMN id SET DEFAULT nextval('ci_secure_files_id_seq'::regclass); +ALTER TABLE ONLY ci_slsa_attestations ALTER COLUMN id SET DEFAULT nextval('ci_slsa_attestations_id_seq'::regclass); + ALTER TABLE ONLY ci_sources_pipelines ALTER COLUMN id SET DEFAULT nextval('ci_sources_pipelines_id_seq'::regclass); ALTER TABLE ONLY ci_sources_projects ALTER COLUMN id SET DEFAULT nextval('ci_sources_projects_id_seq'::regclass); @@ -30558,6 +30582,9 @@ ALTER TABLE ONLY ci_secure_file_states ALTER TABLE ONLY ci_secure_files ADD CONSTRAINT ci_secure_files_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_slsa_attestations + ADD CONSTRAINT ci_slsa_attestations_pkey PRIMARY KEY (id); + ALTER TABLE ONLY ci_sources_pipelines ADD CONSTRAINT ci_sources_pipelines_pkey PRIMARY KEY (id); @@ -34841,6 +34868,8 @@ CREATE INDEX idx_unarchived_occurrences_for_aggregation_severity_nulls_first ON CREATE UNIQUE INDEX idx_uniq_analytics_dashboards_pointers_on_project_id ON analytics_dashboards_pointers USING btree (project_id); +CREATE UNIQUE INDEX idx_uniq_slsa_attestations_on_digest_project_id_predicate_kind ON ci_slsa_attestations USING btree (subject_digest, project_id, predicate_kind); + CREATE UNIQUE INDEX idx_unique_ai_code_repository_connection_namespace_id ON ONLY p_ai_active_context_code_enabled_namespaces USING btree (connection_id, namespace_id); CREATE UNIQUE INDEX idx_unique_ai_code_repository_connection_project_id ON ONLY p_ai_active_context_code_repositories USING btree (connection_id, project_id); diff --git a/spec/factories/ci/slsa/attestations.rb b/spec/factories/ci/slsa/attestations.rb new file mode 100644 index 00000000000000..ebc45835203d6d --- /dev/null +++ b/spec/factories/ci/slsa/attestations.rb @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +FactoryBot.define do + factory :slsa_attestation, class: 'Ci::Slsa::Attestation' do + project factory: :project + predicate_kind { :provenance } + predicate_type { "https://slsa.dev/provenance/v1" } + subject_digest { Digest::SHA256.hexdigest("abc") } + end +end diff --git a/spec/models/ci/slsa/attestation_spec.rb b/spec/models/ci/slsa/attestation_spec.rb new file mode 100644 index 00000000000000..22023d65ef48cd --- /dev/null +++ b/spec/models/ci/slsa/attestation_spec.rb @@ -0,0 +1,17 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Ci::Slsa::Attestation, feature_category: :artifact_security do + describe "validations" do + subject { build(:slsa_attestation) } + + it { is_expected.to belong_to(:project).required } + + it { is_expected.to validate_presence_of(:predicate_kind) } + it { is_expected.to validate_presence_of(:predicate_type) } + it { is_expected.to validate_presence_of(:subject_digest) } + + it { is_expected.to validate_uniqueness_of(:subject_digest).scoped_to([:project_id, :predicate_kind]) } + end +end -- GitLab From 8a62fe0bf0911eff13db7af5ce1b5e8da96b9705 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Tue, 22 Jul 2025 03:50:57 -0500 Subject: [PATCH 02/13] Remove by_digest scope --- app/models/ci/slsa/attestation.rb | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/models/ci/slsa/attestation.rb b/app/models/ci/slsa/attestation.rb index 77e74315c74209..798f1fd916d499 100644 --- a/app/models/ci/slsa/attestation.rb +++ b/app/models/ci/slsa/attestation.rb @@ -23,8 +23,6 @@ class Attestation < SecApplicationRecord provenance: 0, sbom: 1 } - - scope :by_digest, ->(digest) { where(digest: digest) } end end end -- GitLab From e3558c8f1e404008a4e6e505fa522074a8c91999 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Tue, 22 Jul 2025 09:02:15 -0500 Subject: [PATCH 03/13] Add ci_slsa_attestations.project_id to loose foreign keys --- config/gitlab_loose_foreign_keys.yml | 4 ++++ db/migrate/20250714052826_create_ci_slsa_attestations.rb | 5 +++-- db/structure.sql | 2 ++ 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index fbac08f7ac0e65..64c094490d1ba8 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -211,6 +211,10 @@ ci_secure_files: - table: projects column: project_id on_delete: async_delete +ci_slsa_attestations: + - table: projects + column: project_id + on_delete: async_delete ci_sources_pipelines: - table: projects column: source_project_id diff --git a/db/migrate/20250714052826_create_ci_slsa_attestations.rb b/db/migrate/20250714052826_create_ci_slsa_attestations.rb index 394a42052b0e57..1e43afc2982e58 100644 --- a/db/migrate/20250714052826_create_ci_slsa_attestations.rb +++ b/db/migrate/20250714052826_create_ci_slsa_attestations.rb @@ -3,7 +3,7 @@ class CreateCiSlsaAttestations < Gitlab::Database::Migration[2.3] milestone '18.3' - INDEX_NAME = 'idx_uniq_slsa_attestations_on_digest_project_id_predicate_kind' + DIGEST_INDEX_NAME = 'idx_uniq_slsa_attestations_on_digest_project_id_predicate_kind' def change create_table :ci_slsa_attestations do |t| @@ -15,7 +15,8 @@ def change t.text :predicate_type, null: false, limit: 255 end + add_index :ci_slsa_attestations, :project_id add_index :ci_slsa_attestations, [:subject_digest, :project_id, :predicate_kind], - name: INDEX_NAME, unique: true + name: DIGEST_INDEX_NAME, unique: true end end diff --git a/db/structure.sql b/db/structure.sql index 5029967fbaee8b..9dd18b4dd5a1c3 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -35782,6 +35782,8 @@ CREATE INDEX index_ci_secure_file_states_pending_verification ON ci_secure_file_ CREATE INDEX index_ci_secure_files_on_project_id ON ci_secure_files USING btree (project_id); +CREATE INDEX index_ci_slsa_attestations_on_project_id ON ci_slsa_attestations USING btree (project_id); + CREATE INDEX index_ci_sources_pipelines_on_pipeline_id ON ci_sources_pipelines USING btree (pipeline_id); CREATE INDEX index_ci_sources_pipelines_on_project_id ON ci_sources_pipelines USING btree (project_id); -- GitLab From f820b467e4380011694adb86019c7aff7707d692 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Wed, 23 Jul 2025 16:25:29 -0500 Subject: [PATCH 04/13] move to gitlab_ci schema --- db/docs/ci_slsa_attestations.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/db/docs/ci_slsa_attestations.yml b/db/docs/ci_slsa_attestations.yml index bc65f7ceef99c9..4e5a46c303c8af 100644 --- a/db/docs/ci_slsa_attestations.yml +++ b/db/docs/ci_slsa_attestations.yml @@ -9,5 +9,5 @@ classes: - Ci::Slsa::Attestation sharding_key: project_id: projects -gitlab_schema: gitlab_sec +gitlab_schema: gitlab_ci table_size: small \ No newline at end of file -- GitLab From 55490247ed00049f186bb3cdc883b41c572cae0c Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Fri, 25 Jul 2025 12:44:03 -0500 Subject: [PATCH 05/13] Simplify digest index --- db/migrate/20250714052826_create_ci_slsa_attestations.rb | 5 +---- db/structure.sql | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/db/migrate/20250714052826_create_ci_slsa_attestations.rb b/db/migrate/20250714052826_create_ci_slsa_attestations.rb index 1e43afc2982e58..288ea86c22d528 100644 --- a/db/migrate/20250714052826_create_ci_slsa_attestations.rb +++ b/db/migrate/20250714052826_create_ci_slsa_attestations.rb @@ -3,8 +3,6 @@ class CreateCiSlsaAttestations < Gitlab::Database::Migration[2.3] milestone '18.3' - DIGEST_INDEX_NAME = 'idx_uniq_slsa_attestations_on_digest_project_id_predicate_kind' - def change create_table :ci_slsa_attestations do |t| t.integer :status, null: false, default: 0, limit: 2 @@ -16,7 +14,6 @@ def change end add_index :ci_slsa_attestations, :project_id - add_index :ci_slsa_attestations, [:subject_digest, :project_id, :predicate_kind], - name: DIGEST_INDEX_NAME, unique: true + add_index :ci_slsa_attestations, :subject_digest end end diff --git a/db/structure.sql b/db/structure.sql index 9dd18b4dd5a1c3..23f36d507e5c77 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -34868,8 +34868,6 @@ CREATE INDEX idx_unarchived_occurrences_for_aggregation_severity_nulls_first ON CREATE UNIQUE INDEX idx_uniq_analytics_dashboards_pointers_on_project_id ON analytics_dashboards_pointers USING btree (project_id); -CREATE UNIQUE INDEX idx_uniq_slsa_attestations_on_digest_project_id_predicate_kind ON ci_slsa_attestations USING btree (subject_digest, project_id, predicate_kind); - CREATE UNIQUE INDEX idx_unique_ai_code_repository_connection_namespace_id ON ONLY p_ai_active_context_code_enabled_namespaces USING btree (connection_id, namespace_id); CREATE UNIQUE INDEX idx_unique_ai_code_repository_connection_project_id ON ONLY p_ai_active_context_code_repositories USING btree (connection_id, project_id); @@ -35784,6 +35782,8 @@ CREATE INDEX index_ci_secure_files_on_project_id ON ci_secure_files USING btree CREATE INDEX index_ci_slsa_attestations_on_project_id ON ci_slsa_attestations USING btree (project_id); +CREATE INDEX index_ci_slsa_attestations_on_subject_digest ON ci_slsa_attestations USING btree (subject_digest); + CREATE INDEX index_ci_sources_pipelines_on_pipeline_id ON ci_sources_pipelines USING btree (pipeline_id); CREATE INDEX index_ci_sources_pipelines_on_project_id ON ci_sources_pipelines USING btree (project_id); -- GitLab From eb67ee54cdf888c24e08515ac33505e4010651e3 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Sun, 27 Jul 2025 09:45:07 -0500 Subject: [PATCH 06/13] Fix model parent class --- app/models/ci/slsa/attestation.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/ci/slsa/attestation.rb b/app/models/ci/slsa/attestation.rb index 798f1fd916d499..008e50f5cb085f 100644 --- a/app/models/ci/slsa/attestation.rb +++ b/app/models/ci/slsa/attestation.rb @@ -2,7 +2,7 @@ module Ci module Slsa - class Attestation < SecApplicationRecord + class Attestation < Ci::ApplicationRecord self.table_name = 'ci_slsa_attestations' belongs_to :project, optional: false -- GitLab From 41a9d47d433edde35739be10366cf37965140ab4 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Wed, 30 Jul 2025 11:41:50 -0500 Subject: [PATCH 07/13] Optimize column order; update introduced_by_url --- db/docs/ci_slsa_attestations.yml | 2 +- db/migrate/20250714052826_create_ci_slsa_attestations.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/db/docs/ci_slsa_attestations.yml b/db/docs/ci_slsa_attestations.yml index 4e5a46c303c8af..db9b84e43d916f 100644 --- a/db/docs/ci_slsa_attestations.yml +++ b/db/docs/ci_slsa_attestations.yml @@ -1,7 +1,7 @@ --- table_name: ci_slsa_attestations description: Stores SLSA attestations generated by the GitLab Trusted Control Plane -introduced_by_url: TBD +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/198687 milestone: '18.3' feature_categories: - artifact_security diff --git a/db/migrate/20250714052826_create_ci_slsa_attestations.rb b/db/migrate/20250714052826_create_ci_slsa_attestations.rb index 288ea86c22d528..c6fae6b087ee87 100644 --- a/db/migrate/20250714052826_create_ci_slsa_attestations.rb +++ b/db/migrate/20250714052826_create_ci_slsa_attestations.rb @@ -5,12 +5,12 @@ class CreateCiSlsaAttestations < Gitlab::Database::Migration[2.3] def change create_table :ci_slsa_attestations do |t| - t.integer :status, null: false, default: 0, limit: 2 - t.text :subject_digest, null: false, limit: 255 t.timestamps_with_timezone null: false t.bigint :project_id, null: false + t.integer :status, null: false, default: 0, limit: 2 t.integer :predicate_kind, null: false, default: 0, limit: 2 t.text :predicate_type, null: false, limit: 255 + t.text :subject_digest, null: false, limit: 255 end add_index :ci_slsa_attestations, :project_id -- GitLab From bdc9c12d9e04ae561f9442142d809f3b4db19b6e Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Wed, 6 Aug 2025 00:23:13 -0500 Subject: [PATCH 08/13] Partition attestation table --- app/models/ci/slsa/attestation.rb | 11 ++-- .../concerns/ci/partitionable/testing.rb | 1 + config/gitlab_loose_foreign_keys.yml | 8 +-- ...tations.yml => p_ci_slsa_attestations.yml} | 2 +- ...50714052826_create_ci_slsa_attestations.rb | 40 ++++++++++++-- ...e_partitions_for_p_ci_slsa_attestations.rb | 38 +++++++++++++ db/schema_migrations/20250806050946 | 1 + db/structure.sql | 54 +++++++++---------- spec/factories/ci/slsa/attestations.rb | 1 + spec/models/ci/slsa/attestation_spec.rb | 4 +- 10 files changed, 115 insertions(+), 45 deletions(-) rename db/docs/{ci_slsa_attestations.yml => p_ci_slsa_attestations.yml} (90%) create mode 100644 db/migrate/20250806050946_create_partitions_for_p_ci_slsa_attestations.rb create mode 100644 db/schema_migrations/20250806050946 diff --git a/app/models/ci/slsa/attestation.rb b/app/models/ci/slsa/attestation.rb index 008e50f5cb085f..1efd58fa2fa571 100644 --- a/app/models/ci/slsa/attestation.rb +++ b/app/models/ci/slsa/attestation.rb @@ -3,16 +3,21 @@ module Ci module Slsa class Attestation < Ci::ApplicationRecord - self.table_name = 'ci_slsa_attestations' + include Ci::Partitionable - belongs_to :project, optional: false + self.table_name = 'p_ci_slsa_attestations' + + belongs_to :project + belongs_to :artifact, class_name: 'Ci::JobArtifact' + + partitionable scope: :artifact, partitioned: true validates :project_id, presence: true validates :predicate_kind, presence: true validates :predicate_type, presence: true validates :subject_digest, presence: true, length: { minimum: 64, maximum: 255 } - validates :subject_digest, uniqueness: { scope: [:project_id, :predicate_kind] } + validates :subject_digest, uniqueness: { scope: [:partition_id, :project_id, :predicate_kind] } enum :status, { success: 0, diff --git a/app/models/concerns/ci/partitionable/testing.rb b/app/models/concerns/ci/partitionable/testing.rb index 6a16b05332aee1..ae3820a4a0c41d 100644 --- a/app/models/concerns/ci/partitionable/testing.rb +++ b/app/models/concerns/ci/partitionable/testing.rb @@ -35,6 +35,7 @@ module Testing Ci::PipelineMessage Ci::PipelineMetadata Ci::PipelineVariable + Ci::Slsa::Attestation Ci::Sources::Pipeline Ci::Sources::Project Ci::Stage diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 64c094490d1ba8..64ac8d3fa53f72 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -211,10 +211,6 @@ ci_secure_files: - table: projects column: project_id on_delete: async_delete -ci_slsa_attestations: - - table: projects - column: project_id - on_delete: async_delete ci_sources_pipelines: - table: projects column: source_project_id @@ -563,6 +559,10 @@ p_ci_runner_machine_builds: - table: projects column: project_id on_delete: async_delete +p_ci_slsa_attestations: + - table: projects + column: project_id + on_delete: async_delete p_ci_stages: - table: projects column: project_id diff --git a/db/docs/ci_slsa_attestations.yml b/db/docs/p_ci_slsa_attestations.yml similarity index 90% rename from db/docs/ci_slsa_attestations.yml rename to db/docs/p_ci_slsa_attestations.yml index db9b84e43d916f..f1367dd1a35d68 100644 --- a/db/docs/ci_slsa_attestations.yml +++ b/db/docs/p_ci_slsa_attestations.yml @@ -1,5 +1,5 @@ --- -table_name: ci_slsa_attestations +table_name: p_ci_slsa_attestations description: Stores SLSA attestations generated by the GitLab Trusted Control Plane introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/198687 milestone: '18.3' diff --git a/db/migrate/20250714052826_create_ci_slsa_attestations.rb b/db/migrate/20250714052826_create_ci_slsa_attestations.rb index c6fae6b087ee87..dca84265a88446 100644 --- a/db/migrate/20250714052826_create_ci_slsa_attestations.rb +++ b/db/migrate/20250714052826_create_ci_slsa_attestations.rb @@ -1,19 +1,51 @@ # frozen_string_literal: true class CreateCiSlsaAttestations < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers + + disable_ddl_transaction! + milestone '18.3' - def change - create_table :ci_slsa_attestations do |t| + INDEX_NAME = 'index_ci_slsa_attestations_on_digest_project_predicate_uniq' + + def up + opts = { + primary_key: [:id, :partition_id], + options: 'PARTITION BY LIST (partition_id)', + if_not_exists: true + } + + create_table :p_ci_slsa_attestations, **opts do |t| + t.bigserial :id, null: false t.timestamps_with_timezone null: false + t.bigint :partition_id, null: false t.bigint :project_id, null: false + t.bigint :artifact_id, null: false t.integer :status, null: false, default: 0, limit: 2 + t.datetime_with_timezone :expire_at t.integer :predicate_kind, null: false, default: 0, limit: 2 t.text :predicate_type, null: false, limit: 255 t.text :subject_digest, null: false, limit: 255 + + t.index :project_id end - add_index :ci_slsa_attestations, :project_id - add_index :ci_slsa_attestations, :subject_digest + add_index :p_ci_slsa_attestations, [:subject_digest, :partition_id, :project_id, :predicate_kind], + unique: true, + name: INDEX_NAME + + add_concurrent_partitioned_foreign_key( + :p_ci_slsa_attestations, :p_ci_job_artifacts, + column: [:partition_id, :artifact_id], + target_column: [:partition_id, :id], + on_update: :cascade, + on_delete: :nullify, + reverse_lock_order: true + ) + end + + def down + drop_table :p_ci_slsa_attestations end end diff --git a/db/migrate/20250806050946_create_partitions_for_p_ci_slsa_attestations.rb b/db/migrate/20250806050946_create_partitions_for_p_ci_slsa_attestations.rb new file mode 100644 index 00000000000000..28ee1e4852d4f9 --- /dev/null +++ b/db/migrate/20250806050946_create_partitions_for_p_ci_slsa_attestations.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true + +class CreatePartitionsForPCiSlsaAttestations < Gitlab::Database::Migration[2.3] + milestone '18.3' + + disable_ddl_transaction! + + def up + with_lock_retries do + connection.execute(<<~SQL) + LOCK TABLE p_ci_job_artifacts IN SHARE ROW EXCLUSIVE MODE; + LOCK TABLE ONLY p_ci_slsa_attestations IN ACCESS EXCLUSIVE MODE; + SQL + + connection.execute(<<~SQL) + CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_100 + PARTITION OF p_ci_slsa_attestations + FOR VALUES IN (100); + + CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_101 + PARTITION OF p_ci_slsa_attestations + FOR VALUES IN (101); + + CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_102 + PARTITION OF p_ci_slsa_attestations + FOR VALUES IN (102); + SQL + end + end + + def down + connection.execute(<<~SQL) + DROP TABLE IF EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_100; + DROP TABLE IF EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_101; + DROP TABLE IF EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_102; + SQL + end +end diff --git a/db/schema_migrations/20250806050946 b/db/schema_migrations/20250806050946 new file mode 100644 index 00000000000000..c3ce2b7a2e6eda --- /dev/null +++ b/db/schema_migrations/20250806050946 @@ -0,0 +1 @@ +aac70f75f26c368dbc9736fede7f990db976d799adb3412ec02d17349d3bbcc4 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 23f36d507e5c77..609b94b52d94c1 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -4883,6 +4883,23 @@ CREATE TABLE p_ci_runner_machine_builds ( ) PARTITION BY LIST (partition_id); +CREATE TABLE p_ci_slsa_attestations ( + id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + partition_id bigint NOT NULL, + project_id bigint NOT NULL, + artifact_id bigint NOT NULL, + status smallint DEFAULT 0 NOT NULL, + expire_at timestamp with time zone, + predicate_kind smallint DEFAULT 0 NOT NULL, + predicate_type text NOT NULL, + subject_digest text NOT NULL, + CONSTRAINT check_8e9edc8283 CHECK ((char_length(predicate_type) <= 255)), + CONSTRAINT check_90d33eb064 CHECK ((char_length(subject_digest) <= 255)) +) +PARTITION BY LIST (partition_id); + CREATE TABLE p_ci_stages ( project_id bigint, created_at timestamp without time zone, @@ -12526,28 +12543,6 @@ CREATE SEQUENCE ci_secure_files_id_seq ALTER SEQUENCE ci_secure_files_id_seq OWNED BY ci_secure_files.id; -CREATE TABLE ci_slsa_attestations ( - id bigint NOT NULL, - status smallint DEFAULT 0 NOT NULL, - subject_digest text NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - project_id bigint NOT NULL, - predicate_kind smallint DEFAULT 0 NOT NULL, - predicate_type text NOT NULL, - CONSTRAINT check_7966ba494c CHECK ((char_length(subject_digest) <= 255)), - CONSTRAINT check_85b36b80e0 CHECK ((char_length(predicate_type) <= 255)) -); - -CREATE SEQUENCE ci_slsa_attestations_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE ci_slsa_attestations_id_seq OWNED BY ci_slsa_attestations.id; - CREATE TABLE ci_sources_pipelines ( id bigint NOT NULL, project_id bigint, @@ -28204,8 +28199,6 @@ ALTER TABLE ONLY ci_secure_file_states ALTER COLUMN ci_secure_file_id SET DEFAUL ALTER TABLE ONLY ci_secure_files ALTER COLUMN id SET DEFAULT nextval('ci_secure_files_id_seq'::regclass); -ALTER TABLE ONLY ci_slsa_attestations ALTER COLUMN id SET DEFAULT nextval('ci_slsa_attestations_id_seq'::regclass); - ALTER TABLE ONLY ci_sources_pipelines ALTER COLUMN id SET DEFAULT nextval('ci_sources_pipelines_id_seq'::regclass); ALTER TABLE ONLY ci_sources_projects ALTER COLUMN id SET DEFAULT nextval('ci_sources_projects_id_seq'::regclass); @@ -30582,9 +30575,6 @@ ALTER TABLE ONLY ci_secure_file_states ALTER TABLE ONLY ci_secure_files ADD CONSTRAINT ci_secure_files_pkey PRIMARY KEY (id); -ALTER TABLE ONLY ci_slsa_attestations - ADD CONSTRAINT ci_slsa_attestations_pkey PRIMARY KEY (id); - ALTER TABLE ONLY ci_sources_pipelines ADD CONSTRAINT ci_sources_pipelines_pkey PRIMARY KEY (id); @@ -31635,6 +31625,9 @@ ALTER TABLE ONLY p_ci_pipelines ALTER TABLE ONLY p_ci_runner_machine_builds ADD CONSTRAINT p_ci_runner_machine_builds_pkey PRIMARY KEY (build_id, partition_id); +ALTER TABLE ONLY p_ci_slsa_attestations + ADD CONSTRAINT p_ci_slsa_attestations_pkey PRIMARY KEY (id, partition_id); + ALTER TABLE ONLY p_ci_stages ADD CONSTRAINT p_ci_stages_pkey PRIMARY KEY (id, partition_id); @@ -35780,9 +35773,7 @@ CREATE INDEX index_ci_secure_file_states_pending_verification ON ci_secure_file_ CREATE INDEX index_ci_secure_files_on_project_id ON ci_secure_files USING btree (project_id); -CREATE INDEX index_ci_slsa_attestations_on_project_id ON ci_slsa_attestations USING btree (project_id); - -CREATE INDEX index_ci_slsa_attestations_on_subject_digest ON ci_slsa_attestations USING btree (subject_digest); +CREATE UNIQUE INDEX index_ci_slsa_attestations_on_digest_project_predicate_uniq ON ONLY p_ci_slsa_attestations USING btree (subject_digest, partition_id, project_id, predicate_kind); CREATE INDEX index_ci_sources_pipelines_on_pipeline_id ON ci_sources_pipelines USING btree (pipeline_id); @@ -46173,6 +46164,9 @@ ALTER TABLE ONLY project_auto_devops ALTER TABLE ONLY dora_performance_scores ADD CONSTRAINT fk_rails_455f9acc65 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; +ALTER TABLE p_ci_slsa_attestations + ADD CONSTRAINT fk_rails_4572718aba FOREIGN KEY (partition_id, artifact_id) REFERENCES p_ci_job_artifacts(partition_id, id) ON UPDATE CASCADE ON DELETE SET NULL; + ALTER TABLE ONLY merge_requests_closing_issues ADD CONSTRAINT fk_rails_458eda8667 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; diff --git a/spec/factories/ci/slsa/attestations.rb b/spec/factories/ci/slsa/attestations.rb index ebc45835203d6d..92d41941b66c52 100644 --- a/spec/factories/ci/slsa/attestations.rb +++ b/spec/factories/ci/slsa/attestations.rb @@ -3,6 +3,7 @@ FactoryBot.define do factory :slsa_attestation, class: 'Ci::Slsa::Attestation' do project factory: :project + artifact factory: :ci_job_artifact predicate_kind { :provenance } predicate_type { "https://slsa.dev/provenance/v1" } subject_digest { Digest::SHA256.hexdigest("abc") } diff --git a/spec/models/ci/slsa/attestation_spec.rb b/spec/models/ci/slsa/attestation_spec.rb index 22023d65ef48cd..fe39db0a0ba96c 100644 --- a/spec/models/ci/slsa/attestation_spec.rb +++ b/spec/models/ci/slsa/attestation_spec.rb @@ -6,12 +6,10 @@ describe "validations" do subject { build(:slsa_attestation) } - it { is_expected.to belong_to(:project).required } + it { is_expected.to belong_to(:project) } it { is_expected.to validate_presence_of(:predicate_kind) } it { is_expected.to validate_presence_of(:predicate_type) } it { is_expected.to validate_presence_of(:subject_digest) } - - it { is_expected.to validate_uniqueness_of(:subject_digest).scoped_to([:project_id, :predicate_kind]) } end end -- GitLab From 5c94c28ecb04c917fea811a82117be2957d564ac Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Thu, 7 Aug 2025 00:21:37 -0500 Subject: [PATCH 09/13] Update migration timestamps --- ...stations.rb => 20250807083528_create_ci_slsa_attestations.rb} | 0 ...250807083641_create_partitions_for_p_ci_slsa_attestations.rb} | 0 db/schema_migrations/20250714052826 | 1 - db/schema_migrations/20250806050946 | 1 - db/schema_migrations/20250807083528 | 1 + db/schema_migrations/20250807083641 | 1 + 6 files changed, 2 insertions(+), 2 deletions(-) rename db/migrate/{20250714052826_create_ci_slsa_attestations.rb => 20250807083528_create_ci_slsa_attestations.rb} (100%) rename db/migrate/{20250806050946_create_partitions_for_p_ci_slsa_attestations.rb => 20250807083641_create_partitions_for_p_ci_slsa_attestations.rb} (100%) delete mode 100644 db/schema_migrations/20250714052826 delete mode 100644 db/schema_migrations/20250806050946 create mode 100644 db/schema_migrations/20250807083528 create mode 100644 db/schema_migrations/20250807083641 diff --git a/db/migrate/20250714052826_create_ci_slsa_attestations.rb b/db/migrate/20250807083528_create_ci_slsa_attestations.rb similarity index 100% rename from db/migrate/20250714052826_create_ci_slsa_attestations.rb rename to db/migrate/20250807083528_create_ci_slsa_attestations.rb diff --git a/db/migrate/20250806050946_create_partitions_for_p_ci_slsa_attestations.rb b/db/migrate/20250807083641_create_partitions_for_p_ci_slsa_attestations.rb similarity index 100% rename from db/migrate/20250806050946_create_partitions_for_p_ci_slsa_attestations.rb rename to db/migrate/20250807083641_create_partitions_for_p_ci_slsa_attestations.rb diff --git a/db/schema_migrations/20250714052826 b/db/schema_migrations/20250714052826 deleted file mode 100644 index b353dfb5a86823..00000000000000 --- a/db/schema_migrations/20250714052826 +++ /dev/null @@ -1 +0,0 @@ -fc1498e1f9ffc86c2aa78dc524f70cad7fcf1f5ccf3a0ea88105374c0f157215 \ No newline at end of file diff --git a/db/schema_migrations/20250806050946 b/db/schema_migrations/20250806050946 deleted file mode 100644 index c3ce2b7a2e6eda..00000000000000 --- a/db/schema_migrations/20250806050946 +++ /dev/null @@ -1 +0,0 @@ -aac70f75f26c368dbc9736fede7f990db976d799adb3412ec02d17349d3bbcc4 \ No newline at end of file diff --git a/db/schema_migrations/20250807083528 b/db/schema_migrations/20250807083528 new file mode 100644 index 00000000000000..643509a2c76b42 --- /dev/null +++ b/db/schema_migrations/20250807083528 @@ -0,0 +1 @@ +f5aecf90d3d294141b615980898157fb075560c8069f1d6ea423d30802797d69 \ No newline at end of file diff --git a/db/schema_migrations/20250807083641 b/db/schema_migrations/20250807083641 new file mode 100644 index 00000000000000..74d55b3e8d4a4a --- /dev/null +++ b/db/schema_migrations/20250807083641 @@ -0,0 +1 @@ +51cd4d0d9b664d9c72767da9bdaa6c6393c17102974181ed4741eab7d5c8f6ae \ No newline at end of file -- GitLab From 8587147f54e323826c9b15f8f633dc41fe2500d7 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Thu, 7 Aug 2025 16:27:17 -0500 Subject: [PATCH 10/13] Revert table partitioning --- app/models/ci/slsa/attestation.rb | 9 +-- config/gitlab_loose_foreign_keys.yml | 8 +-- ...estations.yml => ci_slsa_attestations.yml} | 4 +- ...50807083528_create_ci_slsa_attestations.rb | 24 +------- ...e_partitions_for_p_ci_slsa_attestations.rb | 38 ------------- db/schema_migrations/20250807083641 | 1 - db/structure.sql | 55 +++++++++++-------- spec/factories/ci/slsa/attestations.rb | 1 - 8 files changed, 42 insertions(+), 98 deletions(-) rename db/docs/{p_ci_slsa_attestations.yml => ci_slsa_attestations.yml} (86%) delete mode 100644 db/migrate/20250807083641_create_partitions_for_p_ci_slsa_attestations.rb delete mode 100644 db/schema_migrations/20250807083641 diff --git a/app/models/ci/slsa/attestation.rb b/app/models/ci/slsa/attestation.rb index 1efd58fa2fa571..92ce3cfe8ec7d5 100644 --- a/app/models/ci/slsa/attestation.rb +++ b/app/models/ci/slsa/attestation.rb @@ -3,21 +3,16 @@ module Ci module Slsa class Attestation < Ci::ApplicationRecord - include Ci::Partitionable - - self.table_name = 'p_ci_slsa_attestations' + self.table_name = 'ci_slsa_attestations' belongs_to :project - belongs_to :artifact, class_name: 'Ci::JobArtifact' - - partitionable scope: :artifact, partitioned: true validates :project_id, presence: true validates :predicate_kind, presence: true validates :predicate_type, presence: true validates :subject_digest, presence: true, length: { minimum: 64, maximum: 255 } - validates :subject_digest, uniqueness: { scope: [:partition_id, :project_id, :predicate_kind] } + validates :subject_digest, uniqueness: { scope: [:project_id, :predicate_kind] } enum :status, { success: 0, diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 64ac8d3fa53f72..64c094490d1ba8 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -211,6 +211,10 @@ ci_secure_files: - table: projects column: project_id on_delete: async_delete +ci_slsa_attestations: + - table: projects + column: project_id + on_delete: async_delete ci_sources_pipelines: - table: projects column: source_project_id @@ -559,10 +563,6 @@ p_ci_runner_machine_builds: - table: projects column: project_id on_delete: async_delete -p_ci_slsa_attestations: - - table: projects - column: project_id - on_delete: async_delete p_ci_stages: - table: projects column: project_id diff --git a/db/docs/p_ci_slsa_attestations.yml b/db/docs/ci_slsa_attestations.yml similarity index 86% rename from db/docs/p_ci_slsa_attestations.yml rename to db/docs/ci_slsa_attestations.yml index f1367dd1a35d68..677d72b831c6d9 100644 --- a/db/docs/p_ci_slsa_attestations.yml +++ b/db/docs/ci_slsa_attestations.yml @@ -1,5 +1,5 @@ --- -table_name: p_ci_slsa_attestations +table_name: ci_slsa_attestations description: Stores SLSA attestations generated by the GitLab Trusted Control Plane introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/198687 milestone: '18.3' @@ -10,4 +10,4 @@ classes: sharding_key: project_id: projects gitlab_schema: gitlab_ci -table_size: small \ No newline at end of file +table_size: small diff --git a/db/migrate/20250807083528_create_ci_slsa_attestations.rb b/db/migrate/20250807083528_create_ci_slsa_attestations.rb index dca84265a88446..de7f33e2455d1f 100644 --- a/db/migrate/20250807083528_create_ci_slsa_attestations.rb +++ b/db/migrate/20250807083528_create_ci_slsa_attestations.rb @@ -1,27 +1,18 @@ # frozen_string_literal: true class CreateCiSlsaAttestations < Gitlab::Database::Migration[2.3] - include Gitlab::Database::PartitioningMigrationHelpers - - disable_ddl_transaction! - milestone '18.3' INDEX_NAME = 'index_ci_slsa_attestations_on_digest_project_predicate_uniq' def up opts = { - primary_key: [:id, :partition_id], - options: 'PARTITION BY LIST (partition_id)', if_not_exists: true } - create_table :p_ci_slsa_attestations, **opts do |t| - t.bigserial :id, null: false + create_table :ci_slsa_attestations, **opts do |t| t.timestamps_with_timezone null: false - t.bigint :partition_id, null: false t.bigint :project_id, null: false - t.bigint :artifact_id, null: false t.integer :status, null: false, default: 0, limit: 2 t.datetime_with_timezone :expire_at t.integer :predicate_kind, null: false, default: 0, limit: 2 @@ -31,21 +22,12 @@ def up t.index :project_id end - add_index :p_ci_slsa_attestations, [:subject_digest, :partition_id, :project_id, :predicate_kind], + add_index :ci_slsa_attestations, [:subject_digest, :project_id, :predicate_kind], unique: true, name: INDEX_NAME - - add_concurrent_partitioned_foreign_key( - :p_ci_slsa_attestations, :p_ci_job_artifacts, - column: [:partition_id, :artifact_id], - target_column: [:partition_id, :id], - on_update: :cascade, - on_delete: :nullify, - reverse_lock_order: true - ) end def down - drop_table :p_ci_slsa_attestations + drop_table :ci_slsa_attestations end end diff --git a/db/migrate/20250807083641_create_partitions_for_p_ci_slsa_attestations.rb b/db/migrate/20250807083641_create_partitions_for_p_ci_slsa_attestations.rb deleted file mode 100644 index 28ee1e4852d4f9..00000000000000 --- a/db/migrate/20250807083641_create_partitions_for_p_ci_slsa_attestations.rb +++ /dev/null @@ -1,38 +0,0 @@ -# frozen_string_literal: true - -class CreatePartitionsForPCiSlsaAttestations < Gitlab::Database::Migration[2.3] - milestone '18.3' - - disable_ddl_transaction! - - def up - with_lock_retries do - connection.execute(<<~SQL) - LOCK TABLE p_ci_job_artifacts IN SHARE ROW EXCLUSIVE MODE; - LOCK TABLE ONLY p_ci_slsa_attestations IN ACCESS EXCLUSIVE MODE; - SQL - - connection.execute(<<~SQL) - CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_100 - PARTITION OF p_ci_slsa_attestations - FOR VALUES IN (100); - - CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_101 - PARTITION OF p_ci_slsa_attestations - FOR VALUES IN (101); - - CREATE TABLE IF NOT EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_102 - PARTITION OF p_ci_slsa_attestations - FOR VALUES IN (102); - SQL - end - end - - def down - connection.execute(<<~SQL) - DROP TABLE IF EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_100; - DROP TABLE IF EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_101; - DROP TABLE IF EXISTS gitlab_partitions_dynamic.ci_slsa_attestations_102; - SQL - end -end diff --git a/db/schema_migrations/20250807083641 b/db/schema_migrations/20250807083641 deleted file mode 100644 index 74d55b3e8d4a4a..00000000000000 --- a/db/schema_migrations/20250807083641 +++ /dev/null @@ -1 +0,0 @@ -51cd4d0d9b664d9c72767da9bdaa6c6393c17102974181ed4741eab7d5c8f6ae \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 609b94b52d94c1..db81a7e50443a1 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -4883,23 +4883,6 @@ CREATE TABLE p_ci_runner_machine_builds ( ) PARTITION BY LIST (partition_id); -CREATE TABLE p_ci_slsa_attestations ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - partition_id bigint NOT NULL, - project_id bigint NOT NULL, - artifact_id bigint NOT NULL, - status smallint DEFAULT 0 NOT NULL, - expire_at timestamp with time zone, - predicate_kind smallint DEFAULT 0 NOT NULL, - predicate_type text NOT NULL, - subject_digest text NOT NULL, - CONSTRAINT check_8e9edc8283 CHECK ((char_length(predicate_type) <= 255)), - CONSTRAINT check_90d33eb064 CHECK ((char_length(subject_digest) <= 255)) -) -PARTITION BY LIST (partition_id); - CREATE TABLE p_ci_stages ( project_id bigint, created_at timestamp without time zone, @@ -12543,6 +12526,29 @@ CREATE SEQUENCE ci_secure_files_id_seq ALTER SEQUENCE ci_secure_files_id_seq OWNED BY ci_secure_files.id; +CREATE TABLE ci_slsa_attestations ( + id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + project_id bigint NOT NULL, + status smallint DEFAULT 0 NOT NULL, + expire_at timestamp with time zone, + predicate_kind smallint DEFAULT 0 NOT NULL, + predicate_type text NOT NULL, + subject_digest text NOT NULL, + CONSTRAINT check_7966ba494c CHECK ((char_length(subject_digest) <= 255)), + CONSTRAINT check_85b36b80e0 CHECK ((char_length(predicate_type) <= 255)) +); + +CREATE SEQUENCE ci_slsa_attestations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE ci_slsa_attestations_id_seq OWNED BY ci_slsa_attestations.id; + CREATE TABLE ci_sources_pipelines ( id bigint NOT NULL, project_id bigint, @@ -28199,6 +28205,8 @@ ALTER TABLE ONLY ci_secure_file_states ALTER COLUMN ci_secure_file_id SET DEFAUL ALTER TABLE ONLY ci_secure_files ALTER COLUMN id SET DEFAULT nextval('ci_secure_files_id_seq'::regclass); +ALTER TABLE ONLY ci_slsa_attestations ALTER COLUMN id SET DEFAULT nextval('ci_slsa_attestations_id_seq'::regclass); + ALTER TABLE ONLY ci_sources_pipelines ALTER COLUMN id SET DEFAULT nextval('ci_sources_pipelines_id_seq'::regclass); ALTER TABLE ONLY ci_sources_projects ALTER COLUMN id SET DEFAULT nextval('ci_sources_projects_id_seq'::regclass); @@ -30575,6 +30583,9 @@ ALTER TABLE ONLY ci_secure_file_states ALTER TABLE ONLY ci_secure_files ADD CONSTRAINT ci_secure_files_pkey PRIMARY KEY (id); +ALTER TABLE ONLY ci_slsa_attestations + ADD CONSTRAINT ci_slsa_attestations_pkey PRIMARY KEY (id); + ALTER TABLE ONLY ci_sources_pipelines ADD CONSTRAINT ci_sources_pipelines_pkey PRIMARY KEY (id); @@ -31625,9 +31636,6 @@ ALTER TABLE ONLY p_ci_pipelines ALTER TABLE ONLY p_ci_runner_machine_builds ADD CONSTRAINT p_ci_runner_machine_builds_pkey PRIMARY KEY (build_id, partition_id); -ALTER TABLE ONLY p_ci_slsa_attestations - ADD CONSTRAINT p_ci_slsa_attestations_pkey PRIMARY KEY (id, partition_id); - ALTER TABLE ONLY p_ci_stages ADD CONSTRAINT p_ci_stages_pkey PRIMARY KEY (id, partition_id); @@ -35773,7 +35781,9 @@ CREATE INDEX index_ci_secure_file_states_pending_verification ON ci_secure_file_ CREATE INDEX index_ci_secure_files_on_project_id ON ci_secure_files USING btree (project_id); -CREATE UNIQUE INDEX index_ci_slsa_attestations_on_digest_project_predicate_uniq ON ONLY p_ci_slsa_attestations USING btree (subject_digest, partition_id, project_id, predicate_kind); +CREATE UNIQUE INDEX index_ci_slsa_attestations_on_digest_project_predicate_uniq ON ci_slsa_attestations USING btree (subject_digest, project_id, predicate_kind); + +CREATE INDEX index_ci_slsa_attestations_on_project_id ON ci_slsa_attestations USING btree (project_id); CREATE INDEX index_ci_sources_pipelines_on_pipeline_id ON ci_sources_pipelines USING btree (pipeline_id); @@ -46164,9 +46174,6 @@ ALTER TABLE ONLY project_auto_devops ALTER TABLE ONLY dora_performance_scores ADD CONSTRAINT fk_rails_455f9acc65 FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; -ALTER TABLE p_ci_slsa_attestations - ADD CONSTRAINT fk_rails_4572718aba FOREIGN KEY (partition_id, artifact_id) REFERENCES p_ci_job_artifacts(partition_id, id) ON UPDATE CASCADE ON DELETE SET NULL; - ALTER TABLE ONLY merge_requests_closing_issues ADD CONSTRAINT fk_rails_458eda8667 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; diff --git a/spec/factories/ci/slsa/attestations.rb b/spec/factories/ci/slsa/attestations.rb index 92d41941b66c52..ebc45835203d6d 100644 --- a/spec/factories/ci/slsa/attestations.rb +++ b/spec/factories/ci/slsa/attestations.rb @@ -3,7 +3,6 @@ FactoryBot.define do factory :slsa_attestation, class: 'Ci::Slsa::Attestation' do project factory: :project - artifact factory: :ci_job_artifact predicate_kind { :provenance } predicate_type { "https://slsa.dev/provenance/v1" } subject_digest { Digest::SHA256.hexdigest("abc") } -- GitLab From abc66053341563e9688c838c19859673493ea6f6 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Tue, 12 Aug 2025 14:46:35 -0500 Subject: [PATCH 11/13] Move from ci to main --- app/models/ci/slsa/attestation.rb | 4 +- .../concerns/ci/partitionable/testing.rb | 1 - config/gitlab_loose_foreign_keys.yml | 4 -- ...attestations.yml => slsa_attestations.yml} | 4 +- ...0250812213110_create_slsa_attestations.rb} | 14 ++-- db/schema_migrations/20250807083528 | 1 - db/schema_migrations/20250812213110 | 1 + db/structure.sql | 67 ++++++++++--------- 8 files changed, 46 insertions(+), 50 deletions(-) rename db/docs/{ci_slsa_attestations.yml => slsa_attestations.yml} (83%) rename db/migrate/{20250807083528_create_ci_slsa_attestations.rb => 20250812213110_create_slsa_attestations.rb} (56%) delete mode 100644 db/schema_migrations/20250807083528 create mode 100644 db/schema_migrations/20250812213110 diff --git a/app/models/ci/slsa/attestation.rb b/app/models/ci/slsa/attestation.rb index 92ce3cfe8ec7d5..220e3bd302d4ee 100644 --- a/app/models/ci/slsa/attestation.rb +++ b/app/models/ci/slsa/attestation.rb @@ -2,8 +2,8 @@ module Ci module Slsa - class Attestation < Ci::ApplicationRecord - self.table_name = 'ci_slsa_attestations' + class Attestation < ::ApplicationRecord + self.table_name = 'slsa_attestations' belongs_to :project diff --git a/app/models/concerns/ci/partitionable/testing.rb b/app/models/concerns/ci/partitionable/testing.rb index ae3820a4a0c41d..6a16b05332aee1 100644 --- a/app/models/concerns/ci/partitionable/testing.rb +++ b/app/models/concerns/ci/partitionable/testing.rb @@ -35,7 +35,6 @@ module Testing Ci::PipelineMessage Ci::PipelineMetadata Ci::PipelineVariable - Ci::Slsa::Attestation Ci::Sources::Pipeline Ci::Sources::Project Ci::Stage diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index 64c094490d1ba8..fbac08f7ac0e65 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -211,10 +211,6 @@ ci_secure_files: - table: projects column: project_id on_delete: async_delete -ci_slsa_attestations: - - table: projects - column: project_id - on_delete: async_delete ci_sources_pipelines: - table: projects column: source_project_id diff --git a/db/docs/ci_slsa_attestations.yml b/db/docs/slsa_attestations.yml similarity index 83% rename from db/docs/ci_slsa_attestations.yml rename to db/docs/slsa_attestations.yml index 677d72b831c6d9..9cbe622eda3348 100644 --- a/db/docs/ci_slsa_attestations.yml +++ b/db/docs/slsa_attestations.yml @@ -1,5 +1,5 @@ --- -table_name: ci_slsa_attestations +table_name: slsa_attestations description: Stores SLSA attestations generated by the GitLab Trusted Control Plane introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/198687 milestone: '18.3' @@ -9,5 +9,5 @@ classes: - Ci::Slsa::Attestation sharding_key: project_id: projects -gitlab_schema: gitlab_ci +gitlab_schema: gitlab_main_cell table_size: small diff --git a/db/migrate/20250807083528_create_ci_slsa_attestations.rb b/db/migrate/20250812213110_create_slsa_attestations.rb similarity index 56% rename from db/migrate/20250807083528_create_ci_slsa_attestations.rb rename to db/migrate/20250812213110_create_slsa_attestations.rb index de7f33e2455d1f..fe808bda42f180 100644 --- a/db/migrate/20250807083528_create_ci_slsa_attestations.rb +++ b/db/migrate/20250812213110_create_slsa_attestations.rb @@ -1,33 +1,31 @@ # frozen_string_literal: true -class CreateCiSlsaAttestations < Gitlab::Database::Migration[2.3] +class CreateSlsaAttestations < Gitlab::Database::Migration[2.3] milestone '18.3' - INDEX_NAME = 'index_ci_slsa_attestations_on_digest_project_predicate_uniq' + INDEX_NAME = 'index_slsa_attestations_on_digest_project_predicate_uniq' def up opts = { if_not_exists: true } - create_table :ci_slsa_attestations, **opts do |t| + create_table :slsa_attestations, **opts do |t| t.timestamps_with_timezone null: false - t.bigint :project_id, null: false + t.references :project, null: false, index: true, foreign_key: { on_delete: :cascade } t.integer :status, null: false, default: 0, limit: 2 t.datetime_with_timezone :expire_at t.integer :predicate_kind, null: false, default: 0, limit: 2 t.text :predicate_type, null: false, limit: 255 t.text :subject_digest, null: false, limit: 255 - - t.index :project_id end - add_index :ci_slsa_attestations, [:subject_digest, :project_id, :predicate_kind], + add_index :slsa_attestations, [:subject_digest, :project_id, :predicate_kind], unique: true, name: INDEX_NAME end def down - drop_table :ci_slsa_attestations + drop_table :slsa_attestations end end diff --git a/db/schema_migrations/20250807083528 b/db/schema_migrations/20250807083528 deleted file mode 100644 index 643509a2c76b42..00000000000000 --- a/db/schema_migrations/20250807083528 +++ /dev/null @@ -1 +0,0 @@ -f5aecf90d3d294141b615980898157fb075560c8069f1d6ea423d30802797d69 \ No newline at end of file diff --git a/db/schema_migrations/20250812213110 b/db/schema_migrations/20250812213110 new file mode 100644 index 00000000000000..62558bb072e2f6 --- /dev/null +++ b/db/schema_migrations/20250812213110 @@ -0,0 +1 @@ +26e368fec804b62238fc3d33cd68c47137d0d13b2069aa825481374478d3b543 \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index db81a7e50443a1..287e8eee4e4898 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -12526,29 +12526,6 @@ CREATE SEQUENCE ci_secure_files_id_seq ALTER SEQUENCE ci_secure_files_id_seq OWNED BY ci_secure_files.id; -CREATE TABLE ci_slsa_attestations ( - id bigint NOT NULL, - created_at timestamp with time zone NOT NULL, - updated_at timestamp with time zone NOT NULL, - project_id bigint NOT NULL, - status smallint DEFAULT 0 NOT NULL, - expire_at timestamp with time zone, - predicate_kind smallint DEFAULT 0 NOT NULL, - predicate_type text NOT NULL, - subject_digest text NOT NULL, - CONSTRAINT check_7966ba494c CHECK ((char_length(subject_digest) <= 255)), - CONSTRAINT check_85b36b80e0 CHECK ((char_length(predicate_type) <= 255)) -); - -CREATE SEQUENCE ci_slsa_attestations_id_seq - START WITH 1 - INCREMENT BY 1 - NO MINVALUE - NO MAXVALUE - CACHE 1; - -ALTER SEQUENCE ci_slsa_attestations_id_seq OWNED BY ci_slsa_attestations.id; - CREATE TABLE ci_sources_pipelines ( id bigint NOT NULL, project_id bigint, @@ -24003,6 +23980,29 @@ CREATE SEQUENCE slack_integrations_scopes_id_seq ALTER SEQUENCE slack_integrations_scopes_id_seq OWNED BY slack_integrations_scopes.id; +CREATE TABLE slsa_attestations ( + id bigint NOT NULL, + created_at timestamp with time zone NOT NULL, + updated_at timestamp with time zone NOT NULL, + project_id bigint NOT NULL, + status smallint DEFAULT 0 NOT NULL, + expire_at timestamp with time zone, + predicate_kind smallint DEFAULT 0 NOT NULL, + predicate_type text NOT NULL, + subject_digest text NOT NULL, + CONSTRAINT check_dec11b603a CHECK ((char_length(subject_digest) <= 255)), + CONSTRAINT check_ea0d61030d CHECK ((char_length(predicate_type) <= 255)) +); + +CREATE SEQUENCE slsa_attestations_id_seq + START WITH 1 + INCREMENT BY 1 + NO MINVALUE + NO MAXVALUE + CACHE 1; + +ALTER SEQUENCE slsa_attestations_id_seq OWNED BY slsa_attestations.id; + CREATE TABLE smartcard_identities ( id bigint NOT NULL, user_id bigint NOT NULL, @@ -28205,8 +28205,6 @@ ALTER TABLE ONLY ci_secure_file_states ALTER COLUMN ci_secure_file_id SET DEFAUL ALTER TABLE ONLY ci_secure_files ALTER COLUMN id SET DEFAULT nextval('ci_secure_files_id_seq'::regclass); -ALTER TABLE ONLY ci_slsa_attestations ALTER COLUMN id SET DEFAULT nextval('ci_slsa_attestations_id_seq'::regclass); - ALTER TABLE ONLY ci_sources_pipelines ALTER COLUMN id SET DEFAULT nextval('ci_sources_pipelines_id_seq'::regclass); ALTER TABLE ONLY ci_sources_projects ALTER COLUMN id SET DEFAULT nextval('ci_sources_projects_id_seq'::regclass); @@ -29089,6 +29087,8 @@ ALTER TABLE ONLY slack_integrations ALTER COLUMN id SET DEFAULT nextval('slack_i ALTER TABLE ONLY slack_integrations_scopes ALTER COLUMN id SET DEFAULT nextval('slack_integrations_scopes_id_seq'::regclass); +ALTER TABLE ONLY slsa_attestations ALTER COLUMN id SET DEFAULT nextval('slsa_attestations_id_seq'::regclass); + ALTER TABLE ONLY smartcard_identities ALTER COLUMN id SET DEFAULT nextval('smartcard_identities_id_seq'::regclass); ALTER TABLE ONLY snippet_repository_states ALTER COLUMN id SET DEFAULT nextval('snippet_repository_states_id_seq'::regclass); @@ -30583,9 +30583,6 @@ ALTER TABLE ONLY ci_secure_file_states ALTER TABLE ONLY ci_secure_files ADD CONSTRAINT ci_secure_files_pkey PRIMARY KEY (id); -ALTER TABLE ONLY ci_slsa_attestations - ADD CONSTRAINT ci_slsa_attestations_pkey PRIMARY KEY (id); - ALTER TABLE ONLY ci_sources_pipelines ADD CONSTRAINT ci_sources_pipelines_pkey PRIMARY KEY (id); @@ -32260,6 +32257,9 @@ ALTER TABLE ONLY slack_integrations ALTER TABLE ONLY slack_integrations_scopes ADD CONSTRAINT slack_integrations_scopes_pkey PRIMARY KEY (id); +ALTER TABLE ONLY slsa_attestations + ADD CONSTRAINT slsa_attestations_pkey PRIMARY KEY (id); + ALTER TABLE ONLY smartcard_identities ADD CONSTRAINT smartcard_identities_pkey PRIMARY KEY (id); @@ -35781,10 +35781,6 @@ CREATE INDEX index_ci_secure_file_states_pending_verification ON ci_secure_file_ CREATE INDEX index_ci_secure_files_on_project_id ON ci_secure_files USING btree (project_id); -CREATE UNIQUE INDEX index_ci_slsa_attestations_on_digest_project_predicate_uniq ON ci_slsa_attestations USING btree (subject_digest, project_id, predicate_kind); - -CREATE INDEX index_ci_slsa_attestations_on_project_id ON ci_slsa_attestations USING btree (project_id); - CREATE INDEX index_ci_sources_pipelines_on_pipeline_id ON ci_sources_pipelines USING btree (pipeline_id); CREATE INDEX index_ci_sources_pipelines_on_project_id ON ci_sources_pipelines USING btree (project_id); @@ -38675,6 +38671,10 @@ CREATE INDEX index_slack_integrations_on_integration_id ON slack_integrations US CREATE UNIQUE INDEX index_slack_integrations_on_team_id_and_alias ON slack_integrations USING btree (team_id, alias); +CREATE UNIQUE INDEX index_slsa_attestations_on_digest_project_predicate_uniq ON slsa_attestations USING btree (subject_digest, project_id, predicate_kind); + +CREATE INDEX index_slsa_attestations_on_project_id ON slsa_attestations USING btree (project_id); + CREATE UNIQUE INDEX index_smartcard_identities_on_subject_and_issuer ON smartcard_identities USING btree (subject, issuer); CREATE INDEX index_smartcard_identities_on_user_id ON smartcard_identities USING btree (user_id); @@ -46852,6 +46852,9 @@ ALTER TABLE ONLY ml_experiments ALTER TABLE ONLY group_repository_storage_moves ADD CONSTRAINT fk_rails_982bb5daf1 FOREIGN KEY (group_id) REFERENCES namespaces(id) ON DELETE CASCADE; +ALTER TABLE ONLY slsa_attestations + ADD CONSTRAINT fk_rails_9834eb1b5e FOREIGN KEY (project_id) REFERENCES projects(id) ON DELETE CASCADE; + ALTER TABLE ONLY resource_label_events ADD CONSTRAINT fk_rails_9851a00031 FOREIGN KEY (merge_request_id) REFERENCES merge_requests(id) ON DELETE CASCADE; -- GitLab From 5697db5c7fee54c42c33891f4689ff2887aedc00 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Fri, 15 Aug 2025 02:26:22 -0500 Subject: [PATCH 12/13] Update milestone to 18.4; add build_id association --- app/models/ci/slsa/attestation.rb | 1 + config/gitlab_loose_foreign_keys.yml | 4 ++++ db/migrate/20250812213110_create_slsa_attestations.rb | 3 ++- db/structure.sql | 3 +++ spec/factories/ci/slsa/attestations.rb | 1 + spec/models/ci/slsa/attestation_spec.rb | 6 +++++- 6 files changed, 16 insertions(+), 2 deletions(-) diff --git a/app/models/ci/slsa/attestation.rb b/app/models/ci/slsa/attestation.rb index 220e3bd302d4ee..a1431813fff4ce 100644 --- a/app/models/ci/slsa/attestation.rb +++ b/app/models/ci/slsa/attestation.rb @@ -6,6 +6,7 @@ class Attestation < ::ApplicationRecord self.table_name = 'slsa_attestations' belongs_to :project + belongs_to :build, class_name: 'Ci::Build', optional: true validates :project_id, presence: true validates :predicate_kind, presence: true diff --git a/config/gitlab_loose_foreign_keys.yml b/config/gitlab_loose_foreign_keys.yml index fbac08f7ac0e65..ff8d52ba30aaa6 100644 --- a/config/gitlab_loose_foreign_keys.yml +++ b/config/gitlab_loose_foreign_keys.yml @@ -725,6 +725,10 @@ security_trainings: - table: projects column: project_id on_delete: async_delete +slsa_attestations: + - table: p_ci_builds + column: build_id + on_delete: async_delete snippets: - table: organizations column: organization_id diff --git a/db/migrate/20250812213110_create_slsa_attestations.rb b/db/migrate/20250812213110_create_slsa_attestations.rb index fe808bda42f180..d8cf777be354a8 100644 --- a/db/migrate/20250812213110_create_slsa_attestations.rb +++ b/db/migrate/20250812213110_create_slsa_attestations.rb @@ -1,7 +1,7 @@ # frozen_string_literal: true class CreateSlsaAttestations < Gitlab::Database::Migration[2.3] - milestone '18.3' + milestone '18.4' INDEX_NAME = 'index_slsa_attestations_on_digest_project_predicate_uniq' @@ -13,6 +13,7 @@ def up create_table :slsa_attestations, **opts do |t| t.timestamps_with_timezone null: false t.references :project, null: false, index: true, foreign_key: { on_delete: :cascade } + t.bigint :build_id, index: true t.integer :status, null: false, default: 0, limit: 2 t.datetime_with_timezone :expire_at t.integer :predicate_kind, null: false, default: 0, limit: 2 diff --git a/db/structure.sql b/db/structure.sql index 287e8eee4e4898..333b104446eea4 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -23985,6 +23985,7 @@ CREATE TABLE slsa_attestations ( created_at timestamp with time zone NOT NULL, updated_at timestamp with time zone NOT NULL, project_id bigint NOT NULL, + build_id bigint, status smallint DEFAULT 0 NOT NULL, expire_at timestamp with time zone, predicate_kind smallint DEFAULT 0 NOT NULL, @@ -38671,6 +38672,8 @@ CREATE INDEX index_slack_integrations_on_integration_id ON slack_integrations US CREATE UNIQUE INDEX index_slack_integrations_on_team_id_and_alias ON slack_integrations USING btree (team_id, alias); +CREATE INDEX index_slsa_attestations_on_build_id ON slsa_attestations USING btree (build_id); + CREATE UNIQUE INDEX index_slsa_attestations_on_digest_project_predicate_uniq ON slsa_attestations USING btree (subject_digest, project_id, predicate_kind); CREATE INDEX index_slsa_attestations_on_project_id ON slsa_attestations USING btree (project_id); diff --git a/spec/factories/ci/slsa/attestations.rb b/spec/factories/ci/slsa/attestations.rb index ebc45835203d6d..0d14e1c7b642ef 100644 --- a/spec/factories/ci/slsa/attestations.rb +++ b/spec/factories/ci/slsa/attestations.rb @@ -3,6 +3,7 @@ FactoryBot.define do factory :slsa_attestation, class: 'Ci::Slsa::Attestation' do project factory: :project + build factory: [:ci_build, :success] predicate_kind { :provenance } predicate_type { "https://slsa.dev/provenance/v1" } subject_digest { Digest::SHA256.hexdigest("abc") } diff --git a/spec/models/ci/slsa/attestation_spec.rb b/spec/models/ci/slsa/attestation_spec.rb index fe39db0a0ba96c..1f0d640a51247c 100644 --- a/spec/models/ci/slsa/attestation_spec.rb +++ b/spec/models/ci/slsa/attestation_spec.rb @@ -4,12 +4,16 @@ RSpec.describe Ci::Slsa::Attestation, feature_category: :artifact_security do describe "validations" do - subject { build(:slsa_attestation) } + subject { create(:slsa_attestation) } it { is_expected.to belong_to(:project) } + it { is_expected.to belong_to(:build) } + it { is_expected.to validate_presence_of(:predicate_kind) } it { is_expected.to validate_presence_of(:predicate_type) } it { is_expected.to validate_presence_of(:subject_digest) } + + it { is_expected.to validate_uniqueness_of(:subject_digest).scoped_to([:project_id, :predicate_kind]) } end end -- GitLab From 65a44f76e651d779d9367a5f0140e90de6d5692c Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Mon, 25 Aug 2025 16:09:49 +0000 Subject: [PATCH 13/13] Apply 2 suggestion(s) to 1 file(s) Co-authored-by: Max Orefice --- db/docs/slsa_attestations.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/db/docs/slsa_attestations.yml b/db/docs/slsa_attestations.yml index 9cbe622eda3348..b161c1c80ebf2c 100644 --- a/db/docs/slsa_attestations.yml +++ b/db/docs/slsa_attestations.yml @@ -2,12 +2,12 @@ table_name: slsa_attestations description: Stores SLSA attestations generated by the GitLab Trusted Control Plane introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/198687 -milestone: '18.3' +milestone: '18.4' feature_categories: - artifact_security classes: - Ci::Slsa::Attestation sharding_key: project_id: projects -gitlab_schema: gitlab_main_cell +gitlab_schema: gitlab_main_org table_size: small -- GitLab