From 39e1610385028c09c40b226c01dc07902c2b56ce Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Sun, 7 Sep 2025 19:47:44 -0500 Subject: [PATCH 1/3] Add file upload capability to Attestation model Allows attestation files generated by GitLab to be stored as attachments to the `SupplyChain::Attestation` model. Changelog: added --- app/models/supply_chain/attestation.rb | 11 ++++ db/docs/slsa_attestation_uploads.yml | 14 +++++ ...905134037_add_file_to_slsa_attestations.rb | 12 ++++ ...ate_slsa_attestations_uploads_partition.rb | 18 ++++++ db/schema_migrations/20250905134037 | 1 + db/schema_migrations/20250905190541 | 1 + db/structure.sql | 62 +++++++++++++++++++ spec/factories/supply_chain/attestations.rb | 1 + spec/fixtures/supply_chain/attestation.json | 33 ++++++++++ spec/models/supply_chain/attestation_spec.rb | 13 +++- 10 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 db/docs/slsa_attestation_uploads.yml create mode 100644 db/migrate/20250905134037_add_file_to_slsa_attestations.rb create mode 100644 db/migrate/20250905190541_create_slsa_attestations_uploads_partition.rb create mode 100644 db/schema_migrations/20250905134037 create mode 100644 db/schema_migrations/20250905190541 create mode 100644 spec/fixtures/supply_chain/attestation.json diff --git a/app/models/supply_chain/attestation.rb b/app/models/supply_chain/attestation.rb index b0d7bebe66173f..ed8fd97d7db484 100644 --- a/app/models/supply_chain/attestation.rb +++ b/app/models/supply_chain/attestation.rb @@ -11,9 +11,12 @@ class Attestation < ::ApplicationRecord validates :predicate_kind, presence: true validates :predicate_type, presence: true validates :subject_digest, presence: true, length: { minimum: 64, maximum: 255 } + validates :file, presence: true validates :subject_digest, uniqueness: { scope: [:project_id, :predicate_kind] } + mount_uploader :file, AttachmentUploader + enum :status, { success: 0, error: 1 @@ -23,5 +26,13 @@ class Attestation < ::ApplicationRecord provenance: 0, sbom: 1 } + + def uploads_sharding_key + { project_id: project_id } + end + + def retrieve_upload(_identifier, paths) + Upload.find_by(model: self, path: paths) + end end end diff --git a/db/docs/slsa_attestation_uploads.yml b/db/docs/slsa_attestation_uploads.yml new file mode 100644 index 00000000000000..cad95d4cdb5c49 --- /dev/null +++ b/db/docs/slsa_attestation_uploads.yml @@ -0,0 +1,14 @@ +--- +table_name: slsa_attestation_uploads +classes: +- SupplyChain::Attesatation +- Upload +feature_categories: +- artifact_security +description: Stores uploads for SupplyChain::Attestation model +introduced_by_url: +milestone: '18.4' +table_size: small +gitlab_schema: gitlab_main_user +sharding_key: + project_id: projects diff --git a/db/migrate/20250905134037_add_file_to_slsa_attestations.rb b/db/migrate/20250905134037_add_file_to_slsa_attestations.rb new file mode 100644 index 00000000000000..7b496b53ef6a3b --- /dev/null +++ b/db/migrate/20250905134037_add_file_to_slsa_attestations.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +class AddFileToSlsaAttestations < Gitlab::Database::Migration[2.3] + milestone '18.4' + + disable_ddl_transaction! + + def change + add_column :slsa_attestations, :file, :text, null: false, default: '' + add_text_limit :slsa_attestations, :file, 1024 + end +end diff --git a/db/migrate/20250905190541_create_slsa_attestations_uploads_partition.rb b/db/migrate/20250905190541_create_slsa_attestations_uploads_partition.rb new file mode 100644 index 00000000000000..10befe44c80829 --- /dev/null +++ b/db/migrate/20250905190541_create_slsa_attestations_uploads_partition.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true + +class CreateSlsaAttestationsUploadsPartition < Gitlab::Database::Migration[2.3] + include Gitlab::Database::PartitioningMigrationHelpers::TableManagementHelpers + + milestone '18.4' + + def up + create_list_partitions( + 'uploads_9ba88c4165', + { slsa_attestation: "'SupplyChain::Attestation'" }, + '%{partition_name}_uploads') + end + + def down + drop_table :slsa_attestation_uploads + end +end diff --git a/db/schema_migrations/20250905134037 b/db/schema_migrations/20250905134037 new file mode 100644 index 00000000000000..14b0d996866c4a --- /dev/null +++ b/db/schema_migrations/20250905134037 @@ -0,0 +1 @@ +dc727e6fd032b112466d37d87552efe773cddd6c20671c03b3372aac82b7ff5a \ No newline at end of file diff --git a/db/schema_migrations/20250905190541 b/db/schema_migrations/20250905190541 new file mode 100644 index 00000000000000..522f3e04d53d98 --- /dev/null +++ b/db/schema_migrations/20250905190541 @@ -0,0 +1 @@ +b87eda3aa27e0a04fb4656b7ef264c3733d3255e5301d64ea626e7dfe7bbdbca \ No newline at end of file diff --git a/db/structure.sql b/db/structure.sql index 853a7b497d6981..0a86a829b1c33d 100644 --- a/db/structure.sql +++ b/db/structure.sql @@ -25157,6 +25157,27 @@ CREATE SEQUENCE slack_integrations_scopes_id_seq ALTER SEQUENCE slack_integrations_scopes_id_seq OWNED BY slack_integrations_scopes.id; +CREATE TABLE slsa_attestation_uploads ( + id bigint NOT NULL, + size bigint NOT NULL, + model_id bigint NOT NULL, + uploaded_by_user_id bigint, + organization_id bigint, + namespace_id bigint, + project_id bigint, + created_at timestamp without time zone, + store integer DEFAULT 1 NOT NULL, + version integer DEFAULT 1, + path text NOT NULL, + checksum text, + model_type text NOT NULL, + uploader text NOT NULL, + mount_point text, + secret text, + CONSTRAINT check_2849dedce7 CHECK ((char_length(path) <= 511)), + CONSTRAINT check_b888b1df14 CHECK ((char_length(checksum) <= 64)) +); + CREATE TABLE slsa_attestations ( id bigint NOT NULL, created_at timestamp with time zone NOT NULL, @@ -25168,6 +25189,8 @@ CREATE TABLE slsa_attestations ( predicate_kind smallint DEFAULT 0 NOT NULL, predicate_type text NOT NULL, subject_digest text NOT NULL, + file text DEFAULT ''::text NOT NULL, + CONSTRAINT check_3575e9121e CHECK ((char_length(file) <= 1024)), CONSTRAINT check_dec11b603a CHECK ((char_length(subject_digest) <= 255)), CONSTRAINT check_ea0d61030d CHECK ((char_length(predicate_type) <= 255)) ); @@ -29200,6 +29223,8 @@ ALTER TABLE ONLY ci_runners ATTACH PARTITION project_type_ci_runners FOR VALUES ALTER TABLE ONLY uploads_9ba88c4165 ATTACH PARTITION project_uploads FOR VALUES IN ('Project'); +ALTER TABLE ONLY uploads_9ba88c4165 ATTACH PARTITION slsa_attestation_uploads FOR VALUES IN ('SupplyChain::Attestation'); + ALTER TABLE ONLY uploads_9ba88c4165 ATTACH PARTITION snippet_uploads FOR VALUES IN ('Snippet'); ALTER TABLE ONLY uploads_9ba88c4165 ATTACH PARTITION user_permission_export_upload_uploads FOR VALUES IN ('UserPermissionExportUpload'); @@ -33866,6 +33891,9 @@ ALTER TABLE ONLY slack_integrations ALTER TABLE ONLY slack_integrations_scopes ADD CONSTRAINT slack_integrations_scopes_pkey PRIMARY KEY (id); +ALTER TABLE ONLY slsa_attestation_uploads + ADD CONSTRAINT slsa_attestation_uploads_pkey PRIMARY KEY (id, model_type); + ALTER TABLE ONLY slsa_attestations ADD CONSTRAINT slsa_attestations_pkey PRIMARY KEY (id); @@ -42252,6 +42280,22 @@ CREATE UNIQUE INDEX security_findings_uuid_scan_id_partition_number_idx ON ONLY CREATE INDEX security_policy_approval_mr_rule_index_merge_request_id ON approval_merge_request_rules USING btree (merge_request_id) WHERE (report_type = ANY (ARRAY[4, 2, 5])); +CREATE INDEX slsa_attestation_uploads_checksum_idx ON slsa_attestation_uploads USING btree (checksum); + +CREATE INDEX slsa_attestation_uploads_model_id_model_type_uploader_creat_idx ON slsa_attestation_uploads USING btree (model_id, model_type, uploader, created_at); + +CREATE INDEX slsa_attestation_uploads_namespace_id_idx ON slsa_attestation_uploads USING btree (namespace_id); + +CREATE INDEX slsa_attestation_uploads_organization_id_idx ON slsa_attestation_uploads USING btree (organization_id); + +CREATE INDEX slsa_attestation_uploads_project_id_idx ON slsa_attestation_uploads USING btree (project_id); + +CREATE INDEX slsa_attestation_uploads_store_idx ON slsa_attestation_uploads USING btree (store); + +CREATE INDEX slsa_attestation_uploads_uploaded_by_user_id_idx ON slsa_attestation_uploads USING btree (uploaded_by_user_id); + +CREATE INDEX slsa_attestation_uploads_uploader_path_idx ON slsa_attestation_uploads USING btree (uploader, path); + CREATE INDEX snippet_uploads_checksum_idx ON snippet_uploads USING btree (checksum); CREATE INDEX snippet_uploads_model_id_model_type_uploader_created_at_idx ON snippet_uploads USING btree (model_id, model_type, uploader, created_at); @@ -45334,6 +45378,24 @@ ALTER INDEX index_uploads_9ba88c4165_on_uploaded_by_user_id ATTACH PARTITION pro ALTER INDEX index_uploads_9ba88c4165_on_uploader_and_path ATTACH PARTITION project_uploads_uploader_path_idx; +ALTER INDEX index_uploads_9ba88c4165_on_checksum ATTACH PARTITION slsa_attestation_uploads_checksum_idx; + +ALTER INDEX index_uploads_9ba88c4165_on_model_uploader_created_at ATTACH PARTITION slsa_attestation_uploads_model_id_model_type_uploader_creat_idx; + +ALTER INDEX index_uploads_9ba88c4165_on_namespace_id ATTACH PARTITION slsa_attestation_uploads_namespace_id_idx; + +ALTER INDEX index_uploads_9ba88c4165_on_organization_id ATTACH PARTITION slsa_attestation_uploads_organization_id_idx; + +ALTER INDEX uploads_9ba88c4165_pkey ATTACH PARTITION slsa_attestation_uploads_pkey; + +ALTER INDEX index_uploads_9ba88c4165_on_project_id ATTACH PARTITION slsa_attestation_uploads_project_id_idx; + +ALTER INDEX index_uploads_9ba88c4165_on_store ATTACH PARTITION slsa_attestation_uploads_store_idx; + +ALTER INDEX index_uploads_9ba88c4165_on_uploaded_by_user_id ATTACH PARTITION slsa_attestation_uploads_uploaded_by_user_id_idx; + +ALTER INDEX index_uploads_9ba88c4165_on_uploader_and_path ATTACH PARTITION slsa_attestation_uploads_uploader_path_idx; + ALTER INDEX index_uploads_9ba88c4165_on_checksum ATTACH PARTITION snippet_uploads_checksum_idx; ALTER INDEX index_uploads_9ba88c4165_on_model_uploader_created_at ATTACH PARTITION snippet_uploads_model_id_model_type_uploader_created_at_idx; diff --git a/spec/factories/supply_chain/attestations.rb b/spec/factories/supply_chain/attestations.rb index 13e3e1bf47fdcd..185ad39ecb8094 100644 --- a/spec/factories/supply_chain/attestations.rb +++ b/spec/factories/supply_chain/attestations.rb @@ -7,5 +7,6 @@ predicate_kind { :provenance } predicate_type { "https://slsa.dev/provenance/v1" } subject_digest { Digest::SHA256.hexdigest("abc") } + file { fixture_file_upload('spec/fixtures/supply_chain/attestation.json') } end end diff --git a/spec/fixtures/supply_chain/attestation.json b/spec/fixtures/supply_chain/attestation.json new file mode 100644 index 00000000000000..7143e8f5f9b828 --- /dev/null +++ b/spec/fixtures/supply_chain/attestation.json @@ -0,0 +1,33 @@ +{ + "mediaType": "application/vnd.dev.sigstore.bundle.v0.3+json", + "verificationMaterial": { + "certificate": { + "rawBytes": "[example]" + }, + "tlogEntries": [ + { + "logIndex": "12345", + "logId": { + "keyId": "[example]" + }, + "kindVersion": { + "kind": "dsse", + "version": "0.0.1" + }, + "inclusionProof": { + "logIndex": "12345" + }, + "canonicalizedBody": "[example]" + } + ] + }, + "dsseEnvelope": { + "payload": "[example]", + "payloadType": "application/vnd.in-toto+json", + "signatures": [ + { + "sig": "[signature]" + } + ] + } +} diff --git a/spec/models/supply_chain/attestation_spec.rb b/spec/models/supply_chain/attestation_spec.rb index 9e457826f92c97..0439a5e4defe95 100644 --- a/spec/models/supply_chain/attestation_spec.rb +++ b/spec/models/supply_chain/attestation_spec.rb @@ -3,9 +3,9 @@ require 'spec_helper' RSpec.describe SupplyChain::Attestation, feature_category: :artifact_security do - describe "validations" do - subject { create(:supply_chain_attestation) } + subject { create(:supply_chain_attestation) } + describe "validations" do it { is_expected.to belong_to(:project) } it { is_expected.to belong_to(:build) } @@ -13,7 +13,16 @@ it { is_expected.to validate_presence_of(:predicate_kind) } it { is_expected.to validate_presence_of(:predicate_type) } it { is_expected.to validate_presence_of(:subject_digest) } + it { is_expected.to validate_presence_of(:file) } it { is_expected.to validate_uniqueness_of(:subject_digest).scoped_to([:project_id, :predicate_kind]) } end + + describe "uploads" do + it_behaves_like "model with uploads", false do + let(:model_object) { create(:supply_chain_attestation) } + let(:upload_attribute) { :file } + let(:uploader_class) { AttachmentUploader } + end + end end -- GitLab From 0364bcdaded8ec0dc95361a5a26c570e4265da31 Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Sun, 7 Sep 2025 20:20:54 -0500 Subject: [PATCH 2/3] Fix irreversible migration --- db/migrate/20250905134037_add_file_to_slsa_attestations.rb | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/db/migrate/20250905134037_add_file_to_slsa_attestations.rb b/db/migrate/20250905134037_add_file_to_slsa_attestations.rb index 7b496b53ef6a3b..4a223d5496af32 100644 --- a/db/migrate/20250905134037_add_file_to_slsa_attestations.rb +++ b/db/migrate/20250905134037_add_file_to_slsa_attestations.rb @@ -5,8 +5,13 @@ class AddFileToSlsaAttestations < Gitlab::Database::Migration[2.3] disable_ddl_transaction! - def change + def up add_column :slsa_attestations, :file, :text, null: false, default: '' add_text_limit :slsa_attestations, :file, 1024 end + + def down + remove_text_limit :slsa_attestations, :file + remove_column :slsa_attestations, :file, if_exists: true + end end -- GitLab From 18b910644b6e69a3493b2d9e5faf6c1ba11398bf Mon Sep 17 00:00:00 2001 From: Aaron Huntsman Date: Sun, 7 Sep 2025 23:01:25 -0500 Subject: [PATCH 3/3] Update partition exceptions in specs --- db/docs/slsa_attestation_uploads.yml | 6 +++--- spec/db/schema_spec.rb | 1 + spec/lib/gitlab/database/sharding_key_spec.rb | 2 ++ 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/db/docs/slsa_attestation_uploads.yml b/db/docs/slsa_attestation_uploads.yml index cad95d4cdb5c49..0b397a52f5bb3f 100644 --- a/db/docs/slsa_attestation_uploads.yml +++ b/db/docs/slsa_attestation_uploads.yml @@ -1,14 +1,14 @@ --- table_name: slsa_attestation_uploads classes: -- SupplyChain::Attesatation - Upload +- SupplyChain::Attesatation feature_categories: - artifact_security description: Stores uploads for SupplyChain::Attestation model -introduced_by_url: +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/204116 milestone: '18.4' table_size: small -gitlab_schema: gitlab_main_user +gitlab_schema: gitlab_main_org sharding_key: project_id: projects diff --git a/spec/db/schema_spec.rb b/spec/db/schema_spec.rb index 56f92996e86aa4..87e86fe831868e 100644 --- a/spec/db/schema_spec.rb +++ b/spec/db/schema_spec.rb @@ -214,6 +214,7 @@ project_import_export_relation_export_upload_uploads: %w[model_id], project_topic_uploads: %w[model_id], project_uploads: %w[model_id], + slsa_attestation_uploads: %w[model_id], snippet_uploads: %w[model_id], user_permission_export_upload_uploads: %w[model_id], user_uploads: %w[model_id], diff --git a/spec/lib/gitlab/database/sharding_key_spec.rb b/spec/lib/gitlab/database/sharding_key_spec.rb index 42999dd9b63571..ff368e8c37aa6e 100644 --- a/spec/lib/gitlab/database/sharding_key_spec.rb +++ b/spec/lib/gitlab/database/sharding_key_spec.rb @@ -48,6 +48,7 @@ project_import_export_relation_export_upload_uploads.project_id project_topic_uploads.organization_id project_uploads.project_id + slsa_attestation_uploads.project_id snippet_uploads.organization_id vulnerability_export_part_uploads.organization_id vulnerability_export_uploads.organization_id @@ -275,6 +276,7 @@ "project_import_export_relation_export_upload_uploads" => "https://gitlab.com/gitlab-org/gitlab/-/issues/398199", "project_topic_uploads" => "https://gitlab.com/gitlab-org/gitlab/-/issues/398199", "project_uploads" => "https://gitlab.com/gitlab-org/gitlab/-/issues/398199", + "slsa_attestation_uploads" => "https://gitlab.com/gitlab-org/gitlab/-/issues/398199", "snippet_uploads" => "https://gitlab.com/gitlab-org/gitlab/-/issues/398199", "uploads_9ba88c4165" => "https://gitlab.com/gitlab-org/gitlab/-/issues/398199", "user_permission_export_upload_uploads" => "https://gitlab.com/gitlab-org/gitlab/-/issues/398199", -- GitLab