diff --git a/config/gitlab.yml.example b/config/gitlab.yml.example index 745bb6d24e0205732070b36959a47da10140adfa..cc0a07600bc5de3c4530a25135c6157b14df283a 100644 --- a/config/gitlab.yml.example +++ b/config/gitlab.yml.example @@ -207,6 +207,26 @@ production: &base # endpoint: 'http://127.0.0.1:9000' # default: nil # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object' + ## Packages (maven repository so far) + packages: + enabled: true + # The location where build packages are stored (default: shared/packages). + # storage_path: shared/packages + object_store: + enabled: false + remote_directory: packages # The bucket name + # direct_upload: false # Use Object Storage directly for uploads instead of background uploads if enabled (Default: false) + # background_upload: false # Temporary option to limit automatic upload (Default: true) + # proxy_download: false # Passthrough all downloads via GitLab instead of using Redirects to Object Storage + connection: + provider: AWS + aws_access_key_id: AWS_ACCESS_KEY_ID + aws_secret_access_key: AWS_SECRET_ACCESS_KEY + region: us-east-1 + # host: 'localhost' # default: s3.amazonaws.com + # endpoint: 'http://127.0.0.1:9000' # default: nil + # path_style: true # Use 'host/bucket_name/object' instead of 'bucket_name.host/object' + ## GitLab Pages pages: enabled: false diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index e17788dccb468ceb4997f2b9912b4f39d513f091..52e0def11485698c0b5b3bb5fe5067877bdfe83f 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -252,6 +252,17 @@ Settings.uploads['object_store'] = ObjectStoreSettings.parse(Settings.uploads['object_store']) Settings.uploads['object_store']['remote_directory'] ||= 'uploads' +# +# Packages +# +Settings['packages'] ||= Settingslogic.new({}) +Settings.packages['enabled'] = true if Settings.packages['enabled'].nil? +Settings.packages['storage_path'] = Settings.absolute(Settings.packages['storage_path'] || File.join(Settings.shared['path'], "packages")) +# Settings.artifact['path'] is deprecated, use `storage_path` instead +Settings.packages['path'] = Settings.packages['storage_path'] +Settings.packages['max_size'] ||= 100 # in megabytes +Settings.packages['object_store'] = ObjectStoreSettings.parse(Settings.packages['object_store']) + # # Mattermost # diff --git a/config/initializers/mysql_set_length_for_binary_indexes.rb b/config/initializers/mysql_set_length_for_binary_indexes.rb index de0bc5322aac572be9b1925d816463c592256cdf..fca8be89ce8f6d62a587e2e51133dfd788ad5129 100644 --- a/config/initializers/mysql_set_length_for_binary_indexes.rb +++ b/config/initializers/mysql_set_length_for_binary_indexes.rb @@ -1,6 +1,6 @@ -# This patches ActiveRecord so indexes for binary columns created using the -# MySQL adapter apply a length of 20. Otherwise MySQL can't create an index on -# binary columns. +# This patches ActiveRecord so indexes for binary columns created +# using the MySQL adapter apply a length of 20. Otherwise MySQL can't create an +# index on binary columns. module MysqlSetLengthForBinaryIndex def add_index(table_name, column_names, options = {}) diff --git a/db/schema.rb b/db/schema.rb index 5b3527ae2ce79dc59b859abd5af6bc0a0af00791..e2d31702a0a62b892be41c0a7f54d431c312a5c6 100644 --- a/db/schema.rb +++ b/db/schema.rb @@ -1921,6 +1921,43 @@ t.string "nonce", null: false end + create_table "packages_maven_metadata", id: :bigserial, force: :cascade do |t| + t.integer "package_id", limit: 8, null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.string "app_group", null: false + t.string "app_name", null: false + t.string "app_version" + t.string "path", limit: 512, null: false + end + + add_index "packages_maven_metadata", ["package_id", "path"], name: "index_packages_maven_metadata_on_package_id_and_path", using: :btree + + create_table "packages_package_files", id: :bigserial, force: :cascade do |t| + t.integer "package_id", limit: 8, null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.integer "size", limit: 8 + t.integer "file_type" + t.integer "file_store" + t.binary "file_md5" + t.binary "file_sha1" + t.string "file_name", null: false + t.text "file", null: false + end + + add_index "packages_package_files", ["package_id", "file_name"], name: "index_packages_package_files_on_package_id_and_file_name", using: :btree + + create_table "packages_packages", id: :bigserial, force: :cascade do |t| + t.integer "project_id", null: false + t.datetime_with_timezone "created_at", null: false + t.datetime_with_timezone "updated_at", null: false + t.string "name", null: false + t.string "version" + end + + add_index "packages_packages", ["project_id"], name: "index_packages_packages_on_project_id", using: :btree + create_table "pages_domains", force: :cascade do |t| t.integer "project_id" t.text "certificate" @@ -3082,6 +3119,9 @@ add_foreign_key "notes", "projects", name: "fk_99e097b079", on_delete: :cascade add_foreign_key "notification_settings", "users", name: "fk_0c95e91db7", on_delete: :cascade add_foreign_key "oauth_openid_requests", "oauth_access_grants", column: "access_grant_id", name: "fk_oauth_openid_requests_oauth_access_grants_access_grant_id" + add_foreign_key "packages_maven_metadata", "packages_packages", column: "package_id", name: "fk_be88aed360", on_delete: :cascade + add_foreign_key "packages_package_files", "packages_packages", column: "package_id", name: "fk_86f0f182f8", on_delete: :cascade + add_foreign_key "packages_packages", "projects", on_delete: :cascade add_foreign_key "pages_domains", "projects", name: "fk_ea2f6dfc6f", on_delete: :cascade add_foreign_key "path_locks", "projects", name: "fk_5265c98f24", on_delete: :cascade add_foreign_key "path_locks", "users" diff --git a/doc/user/project/maven_packages.md b/doc/user/project/maven_packages.md new file mode 100644 index 0000000000000000000000000000000000000000..3ff70135efaa572ad6c27e2ae08b70c0e8a4d514 --- /dev/null +++ b/doc/user/project/maven_packages.md @@ -0,0 +1,55 @@ +# GitLab Maven Packages repository + +## Configure project to use GitLab Maven Repository URL + +To download packages from GitLab, you need `repository` section in your `pom.xml`. + +```xml + + + gitlab-maven + https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven + + +``` + +To upload packages to GitLab, you need a `distributionManagement` section in your `pom.xml`. + +```xml + + + gitlab-maven + https://gitlab.com/api/v4/projects/PROJECT_ID/packages/maven + + +``` + +In both examples, replace `PROJECT_ID` with your project ID. +If you have a private GitLab installation, replace `gitlab.com` with your domain name. + +## Configure repository access + +If a project is private, credentials will need to be provided for authorization. +The preferred way to do this, is by using a [personal access tokens][pat]. +You can add a corresponding section to your `settings.xml` file: + + +```xml + + + + gitlab-maven + + + + Private-Token + REPLACE_WITH_YOUR_PRIVATE_TOKEN + + + + + + +``` + +[pat]: ../profile/personal_access_tokens.md diff --git a/ee/app/finders/packages/maven_package_finder.rb b/ee/app/finders/packages/maven_package_finder.rb new file mode 100644 index 0000000000000000000000000000000000000000..f2cc5bf0af37b9e1f85be36e55a8f05c3eac8701 --- /dev/null +++ b/ee/app/finders/packages/maven_package_finder.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true +class Packages::MavenPackageFinder + attr_reader :project, :path + + def initialize(project, path) + @project = project + @path = path + end + + def execute + packages.last + end + + def execute! + packages.last! + end + + private + + def packages + project.packages.joins(:maven_metadatum) + .where(packages_maven_metadata: { path: path }) + end +end diff --git a/ee/app/finders/packages/package_file_finder.rb b/ee/app/finders/packages/package_file_finder.rb new file mode 100644 index 0000000000000000000000000000000000000000..74db44073d99e03c74726022bec7471a302475bd --- /dev/null +++ b/ee/app/finders/packages/package_file_finder.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true +class Packages::PackageFileFinder + attr_reader :package, :file_name + + def initialize(package, file_name) + @package = package + @file_name = file_name + end + + def execute + package_files.last + end + + def execute! + package_files.last! + end + + private + + def package_files + package.package_files.where(file_name: file_name) + end +end diff --git a/ee/app/models/ee/project.rb b/ee/app/models/ee/project.rb index 79a14ba64721cd78291c67c85093e818cf99f6ee..1c442047bc8ab627b1b4f6b261d37c8c6650a1d9 100644 --- a/ee/app/models/ee/project.rb +++ b/ee/app/models/ee/project.rb @@ -38,6 +38,7 @@ module Project has_many :protected_environments has_many :software_license_policies, inverse_of: :project, class_name: 'SoftwareLicensePolicy' accepts_nested_attributes_for :software_license_policies, allow_destroy: true + has_many :packages, class_name: 'Packages::Package' has_many :sourced_pipelines, class_name: 'Ci::Sources::Pipeline', foreign_key: :source_project_id diff --git a/ee/app/models/license.rb b/ee/app/models/license.rb index 3c9c4238eaaf29638d12129b266005162e8e71ec..341fb416c917345cdedbe738d287fbc8cd7d688e 100644 --- a/ee/app/models/license.rb +++ b/ee/app/models/license.rb @@ -64,6 +64,7 @@ class License < ActiveRecord::Base protected_environments system_header_footer custom_project_templates + packages ].freeze EEU_FEATURES = EEP_FEATURES + %i[ diff --git a/ee/app/models/packages.rb b/ee/app/models/packages.rb new file mode 100644 index 0000000000000000000000000000000000000000..e14c9290093f256b5fa04b37c696ab964147733a --- /dev/null +++ b/ee/app/models/packages.rb @@ -0,0 +1,6 @@ +# frozen_string_literal: true +module Packages + def self.table_name_prefix + 'packages_' + end +end diff --git a/ee/app/models/packages/maven_metadatum.rb b/ee/app/models/packages/maven_metadatum.rb new file mode 100644 index 0000000000000000000000000000000000000000..e1f9d064987f1a260befc0635b3b2a82bdb6719a --- /dev/null +++ b/ee/app/models/packages/maven_metadatum.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +class Packages::MavenMetadatum < ActiveRecord::Base + belongs_to :package + + validates :package, presence: true + + validates :path, + presence: true, + format: { with: Gitlab::Regex.maven_path_regex } + + validates :app_group, + presence: true, + format: { with: Gitlab::Regex.maven_app_group_regex } + + validates :app_name, + presence: true, + format: { with: Gitlab::Regex.maven_app_name_regex } +end diff --git a/ee/app/models/packages/package.rb b/ee/app/models/packages/package.rb new file mode 100644 index 0000000000000000000000000000000000000000..b593f9546eb4fb291844eb8a31e09acb502d62ef --- /dev/null +++ b/ee/app/models/packages/package.rb @@ -0,0 +1,14 @@ +# frozen_string_literal: true +class Packages::Package < ActiveRecord::Base + belongs_to :project + has_many :package_files + has_one :maven_metadatum, inverse_of: :package + + accepts_nested_attributes_for :maven_metadatum + + validates :project, presence: true + + validates :name, + presence: true, + format: { with: Gitlab::Regex.package_name_regex } +end diff --git a/ee/app/models/packages/package_file.rb b/ee/app/models/packages/package_file.rb new file mode 100644 index 0000000000000000000000000000000000000000..fb2697e7e8246b3d9a67bf6e6af783dec5ff91b9 --- /dev/null +++ b/ee/app/models/packages/package_file.rb @@ -0,0 +1,18 @@ +# frozen_string_literal: true +class Packages::PackageFile < ActiveRecord::Base + belongs_to :package + + validates :package, presence: true + validates :file, presence: true + validates :file_name, presence: true + + mount_uploader :file, Packages::PackageFileUploader + + after_save :update_file_store, if: :file_changed? + + def update_file_store + # The file.object_store is set during `uploader.store!` + # which happens after object is inserted/updated + self.update_column(:file_store, file.object_store) + end +end diff --git a/ee/app/policies/ee/project_policy.rb b/ee/app/policies/ee/project_policy.rb index 640c28f8e23ac7471aef278fc4200bf37884c0f0..8d20b973e1095b8e5a2d9803bb98a0178667d2e6 100644 --- a/ee/app/policies/ee/project_policy.rb +++ b/ee/app/policies/ee/project_policy.rb @@ -89,13 +89,17 @@ module ProjectPolicy enable :read_deploy_board enable :admin_issue_link enable :admin_epic_issue + enable :read_package end rule { can?(:developer_access) }.policy do enable :admin_board enable :admin_vulnerability_feedback + enable :create_package end + rule { can?(:public_access) }.enable :read_package + rule { can?(:developer_access) & security_reports_feature_available }.enable :read_project_security_dashboard rule { can?(:read_project) }.enable :read_vulnerability_feedback diff --git a/ee/app/services/packages/create_maven_package_service.rb b/ee/app/services/packages/create_maven_package_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..672e628ca5ecad16d77664943cd4cfaab0989030 --- /dev/null +++ b/ee/app/services/packages/create_maven_package_service.rb @@ -0,0 +1,20 @@ +# frozen_string_literal: true +module Packages + class CreateMavenPackageService < BaseService + def execute + app_group, _, app_name = params[:name].rpartition('/') + app_group.tr!('/', '.') + + project.packages.create!( + name: params[:name], + version: params[:version], + maven_metadatum_attributes: { + path: params[:path], + app_group: app_group, + app_name: app_name, + app_version: params[:version] + } + ) + end + end +end diff --git a/ee/app/services/packages/create_package_file_service.rb b/ee/app/services/packages/create_package_file_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..58abed5bcd58467daeea133fead815dfa94b5da5 --- /dev/null +++ b/ee/app/services/packages/create_package_file_service.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +module Packages + class CreatePackageFileService + attr_reader :package, :params + + def initialize(package, params) + @package = package + @params = params + end + + def execute + package.package_files.create!( + file: params[:file], + size: params[:size], + file_name: params[:file_name], + file_type: params[:file_type], + file_sha1: params[:file_sha1], + file_md5: params[:file_md5] + ) + end + end +end diff --git a/ee/app/services/packages/find_or_create_maven_package_service.rb b/ee/app/services/packages/find_or_create_maven_package_service.rb new file mode 100644 index 0000000000000000000000000000000000000000..141f31a2169030819978ac3978556b0d2e75ddbc --- /dev/null +++ b/ee/app/services/packages/find_or_create_maven_package_service.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true +module Packages + class FindOrCreateMavenPackageService < BaseService + MAVEN_METADATA_FILE = 'maven-metadata.xml'.freeze + + def execute + package = ::Packages::MavenPackageFinder + .new(project, params[:path]).execute + + unless package + if params[:file_name] == MAVEN_METADATA_FILE + # Maven uploads several files during `mvn deploy` in next order: + # - my-company/my-app/1.0-SNAPSHOT/my-app.jar + # - my-company/my-app/1.0-SNAPSHOT/my-app.pom + # - my-company/my-app/1.0-SNAPSHOT/maven-metadata.xml + # - my-company/my-app/maven-metadata.xml + # + # The last xml file does not have VERSION in URL because it contains + # information about all versions. + package_name, version = params[:path], nil + else + package_name, _, version = params[:path].rpartition('/') + end + + package_params = { + name: package_name, + path: params[:path], + version: version + } + + package = ::Packages::CreateMavenPackageService + .new(project, current_user, package_params).execute + end + + package + end + end +end diff --git a/ee/app/uploaders/packages/package_file_uploader.rb b/ee/app/uploaders/packages/package_file_uploader.rb new file mode 100644 index 0000000000000000000000000000000000000000..6bd3bd7407df1b294f98e70cc53ee35156b34a0c --- /dev/null +++ b/ee/app/uploaders/packages/package_file_uploader.rb @@ -0,0 +1,26 @@ +# frozen_string_literal: true +class Packages::PackageFileUploader < GitlabUploader + extend Workhorse::UploadPath + include ObjectStorage::Concern + + storage_options Gitlab.config.packages + + def filename + model.file_name + end + + def store_dir + dynamic_segment + end + + private + + def dynamic_segment + File.join(disk_hash[0..1], disk_hash[2..3], disk_hash, + 'packages', model.package.id.to_s, 'files', model.id.to_s) + end + + def disk_hash + @disk_hash ||= Digest::SHA2.hexdigest(model.package.project_id.to_s) + end +end diff --git a/ee/changelogs/unreleased/5811-add-maven-support-to-our-artifact-repository-mvc.yml b/ee/changelogs/unreleased/5811-add-maven-support-to-our-artifact-repository-mvc.yml new file mode 100644 index 0000000000000000000000000000000000000000..e75e1855b998167500455a838c7fd4dd2e12779d --- /dev/null +++ b/ee/changelogs/unreleased/5811-add-maven-support-to-our-artifact-repository-mvc.yml @@ -0,0 +1,5 @@ +--- +title: Add ability to upload and download maven packages from/to GitLab +merge_request: 6607 +author: +type: added diff --git a/ee/db/migrate/20180720120716_create_packages_packages.rb b/ee/db/migrate/20180720120716_create_packages_packages.rb new file mode 100644 index 0000000000000000000000000000000000000000..847e5f86a249f7ab843fd8dc8aa18929195aafb2 --- /dev/null +++ b/ee/db/migrate/20180720120716_create_packages_packages.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true +class CreatePackagesPackages < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def change + create_table :packages_packages, id: :bigserial do |t| + t.references :project, + index: true, + foreign_key: { on_delete: :cascade }, + null: false + + t.timestamps_with_timezone null: false + + t.string :name, null: false + t.string :version + end + end +end diff --git a/ee/db/migrate/20180720120726_create_packages_package_files.rb b/ee/db/migrate/20180720120726_create_packages_package_files.rb new file mode 100644 index 0000000000000000000000000000000000000000..307646a0ce258399b9ae1a5bf1cd14f94db92031 --- /dev/null +++ b/ee/db/migrate/20180720120726_create_packages_package_files.rb @@ -0,0 +1,45 @@ +# frozen_string_literal: true +class CreatePackagesPackageFiles < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + create_table :packages_package_files, id: :bigserial do |t| + t.references :package, type: :bigint, null: false + + t.timestamps_with_timezone null: false + + t.bigint :size + t.integer :file_type + t.integer :file_store + t.binary :file_md5 + t.binary :file_sha1 + + t.string :file_name, null: false + t.text :file, null: false + end + + add_concurrent_index :packages_package_files, [:package_id, :file_name] + + add_concurrent_foreign_key :packages_package_files, :packages_packages, + column: :package_id, + on_delete: :cascade + end + + def down + if foreign_keys_for(:packages_package_files, :package_id).any? + remove_foreign_key :packages_package_files, column: :package_id + end + + if index_exists?(:packages_package_files, [:package_id, :file_name]) + remove_concurrent_index :packages_package_files, [:package_id, :file_name] + end + + if table_exists?(:packages_package_files) + drop_table :packages_package_files + end + end +end diff --git a/ee/db/migrate/20180720121404_create_packages_maven_metadata.rb b/ee/db/migrate/20180720121404_create_packages_maven_metadata.rb new file mode 100644 index 0000000000000000000000000000000000000000..f41d02bb65ff24154978385fd4fb8ec91eb4b8df --- /dev/null +++ b/ee/db/migrate/20180720121404_create_packages_maven_metadata.rb @@ -0,0 +1,41 @@ +# frozen_string_literal: true +class CreatePackagesMavenMetadata < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def up + create_table :packages_maven_metadata, id: :bigserial do |t| + t.references :package, type: :bigint, null: false + + t.timestamps_with_timezone null: false + + t.string :app_group, null: false + t.string :app_name, null: false + t.string :app_version + t.string :path, limit: 512, null: false + end + + add_concurrent_index :packages_maven_metadata, [:package_id, :path] + + add_concurrent_foreign_key :packages_maven_metadata, :packages_packages, + column: :package_id, + on_delete: :cascade + end + + def down + if foreign_keys_for(:packages_maven_metadata, :package_id).any? + remove_foreign_key :packages_maven_metadata, column: :package_id + end + + if index_exists?(:packages_maven_metadata, [:package_id, :path]) + remove_concurrent_index :packages_maven_metadata, [:package_id, :path] + end + + if table_exists?(:packages_maven_metadata) + drop_table :packages_maven_metadata + end + end +end diff --git a/ee/lib/api/maven_packages.rb b/ee/lib/api/maven_packages.rb new file mode 100644 index 0000000000000000000000000000000000000000..7a66e860c5d6f8b0ec98ead11ec1667f150bda00 --- /dev/null +++ b/ee/lib/api/maven_packages.rb @@ -0,0 +1,157 @@ +# frozen_string_literal: true +module API + class MavenPackages < Grape::API + MAVEN_ENDPOINT_REQUIREMENTS = { + file_name: API::NO_SLASH_URL_PART_REGEX + }.freeze + + content_type :md5, 'text/plain' + content_type :sha1, 'text/plain' + content_type :binary, 'application/octet-stream' + + before do + require_packages_enabled! + authenticate_non_get! + authorize_packages_feature! + end + + helpers do + def require_packages_enabled! + not_found! unless Gitlab.config.packages.enabled + end + + def authorize_packages_feature! + forbidden! unless user_project.feature_available?(:packages) + end + + def authorize_download_package! + authorize!(:read_package, user_project) + end + + def authorize_create_package! + authorize!(:create_package, user_project) + end + + def extract_format(file_name) + name, _, format = file_name.rpartition('.') + + if %w(md5 sha1).include?(format) + [name, format] + else + [file_name, nil] + end + end + + def verify_package_file(package_file, uploaded_file) + stored_sha1 = Digest::SHA256.hexdigest(package_file.file_sha1) + expected_sha1 = uploaded_file.sha256 + + if stored_sha1 == expected_sha1 + no_content! + else + conflict! + end + end + end + + params do + requires :id, type: String, desc: 'The ID of a project' + end + resource :projects, requirements: API::PROJECT_ENDPOINT_REQUIREMENTS do + desc 'Download the maven package file' do + detail 'This feature was introduced in GitLab 11.3' + end + params do + requires :path, type: String, desc: 'Package path' + requires :file_name, type: String, desc: 'Package file name' + end + get ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do + authorize_download_package! + + file_name, format = extract_format(params[:file_name]) + + package = ::Packages::MavenPackageFinder + .new(user_project, params[:path]).execute! + + package_file = ::Packages::PackageFileFinder + .new(package, file_name).execute! + + case format + when 'md5' + package_file.file_md5 + when 'sha1' + package_file.file_sha1 + when nil + present_carrierwave_file!(package_file.file) + end + end + + desc 'Upload the maven package file' do + detail 'This feature was introduced in GitLab 11.3' + end + params do + requires :path, type: String, desc: 'Package path' + requires :file_name, type: String, desc: 'Package file name' + end + put ':id/packages/maven/*path/:file_name/authorize', requirements: MAVEN_ENDPOINT_REQUIREMENTS do + authorize_create_package! + + require_gitlab_workhorse! + Gitlab::Workhorse.verify_api_request!(headers) + + status 200 + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + ::Packages::PackageFileUploader.workhorse_authorize(has_length: true) + end + + desc 'Upload the maven package file' do + detail 'This feature was introduced in GitLab 11.3' + end + params do + requires :path, type: String, desc: 'Package path' + requires :file_name, type: String, desc: 'Package file name' + optional 'file.path', type: String, desc: %q(path to locally stored body (generated by Workhorse)) + optional 'file.name', type: String, desc: %q(real filename as send in Content-Disposition (generated by Workhorse)) + optional 'file.type', type: String, desc: %q(real content type as send in Content-Type (generated by Workhorse)) + optional 'file.size', type: Integer, desc: %q(real size of file (generated by Workhorse)) + optional 'file.md5', type: String, desc: %q(md5 checksum of the file (generated by Workhorse)) + optional 'file.sha1', type: String, desc: %q(sha1 checksum of the file (generated by Workhorse)) + optional 'file.sha256', type: String, desc: %q(sha256 checksum of the file (generated by Workhorse)) + end + put ':id/packages/maven/*path/:file_name', requirements: MAVEN_ENDPOINT_REQUIREMENTS do + authorize_create_package! + require_gitlab_workhorse! + + file_name, format = extract_format(params[:file_name]) + + uploaded_file = UploadedFile.from_params(params, :file, ::Packages::PackageFileUploader.workhorse_local_upload_path) + bad_request!('Missing package file!') unless uploaded_file + + package = ::Packages::FindOrCreateMavenPackageService + .new(user_project, current_user, params).execute + + case format + when 'sha1' + # After uploading a file, Maven tries to upload a sha1 and md5 version of it. + # Since we store md5/sha1 in database we simply need to validate our hash + # against one uploaded by Maven. We do this for `sha1` format. + package_file = ::Packages::PackageFileFinder + .new(package, file_name).execute! + + verify_package_file(package_file, uploaded_file) + when nil + file_params = { + file: uploaded_file, + size: params['file.size'], + file_name: file_name, + file_type: params['file.type'], + file_sha1: params['file.sha1'], + file_md5: params['file.md5'] + } + + ::Packages::CreatePackageFileService.new(package, file_params).execute + end + end + end + end +end diff --git a/ee/lib/ee/gitlab/regex.rb b/ee/lib/ee/gitlab/regex.rb index 6747bf7ebe59fb259a90e59576510a5755f07b0b..82e081bfac44d0cede5648fa456e055264c9f7aa 100644 --- a/ee/lib/ee/gitlab/regex.rb +++ b/ee/lib/ee/gitlab/regex.rb @@ -12,6 +12,22 @@ def environment_scope_regex def environment_scope_regex_message "can contain only letters, digits, '-', '_', '/', '$', '{', '}', '.', '*' and spaces" end + + def package_name_regex + @package_name_regex ||= %r{\A(([\w\-\.]*)/)*([\w\-\.]*)\z}.freeze + end + + def maven_path_regex + package_name_regex + end + + def maven_app_name_regex + @maven_app_name_regex ||= /\A[\w\-\.]+\z/.freeze + end + + def maven_app_group_regex + maven_app_name_regex + end end end end diff --git a/ee/spec/factories/packages.rb b/ee/spec/factories/packages.rb new file mode 100644 index 0000000000000000000000000000000000000000..c89cb13a42a941838c52cace9266ec7eb28f0a0b --- /dev/null +++ b/ee/spec/factories/packages.rb @@ -0,0 +1,51 @@ +# frozen_string_literal: true +FactoryBot.define do + factory :package, class: Packages::Package do + project + name 'my/company/app/my-app' + version '1-0-SNAPSHOT' + + factory :maven_package do + maven_metadatum + + after :create do |package| + create :package_file, :xml, package: package + create :package_file, :jar, package: package + create :package_file, :pom, package: package + end + end + end + + factory :package_file, class: Packages::PackageFile do + package + + trait(:jar) do + file { fixture_file_upload('ee/spec/fixtures/maven/my-app-1.0-20180724.124855-1.jar') } + file_name 'my-app-1.0-20180724.124855-1.jar' + file_sha1 '4f0bfa298744d505383fbb57c554d4f5c12d88b3' + file_type 'jar' + end + + trait(:pom) do + file { fixture_file_upload('ee/spec/fixtures/maven/my-app-1.0-20180724.124855-1.pom') } + file_name 'my-app-1.0-20180724.124855-1.pom' + file_sha1 '19c975abd49e5102ca6c74a619f21e0cf0351c57' + file_type 'pom' + end + + trait(:xml) do + file { fixture_file_upload('ee/spec/fixtures/maven/maven-metadata.xml') } + file_name 'maven-metadata.xml' + file_sha1 '42b1bdc80de64953b6876f5a8c644f20204011b0' + file_type 'xml' + end + end + + factory :maven_metadatum, class: Packages::MavenMetadatum do + package + path 'my/company/app/my-app/1.0-SNAPSHOT' + app_group 'my.company.app' + app_name 'my-app' + app_version '1.0-SNAPSHOT' + end +end diff --git a/ee/spec/finders/packages/maven_package_finder_spec.rb b/ee/spec/finders/packages/maven_package_finder_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..eb198f527c2e48caceca747df23838f6f00a9332 --- /dev/null +++ b/ee/spec/finders/packages/maven_package_finder_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Packages::MavenPackageFinder do + let(:project) { create(:project) } + let(:package) { create(:maven_package, project: project) } + + describe '#execute!' do + it 'returns a package' do + finder = described_class.new(project, package.maven_metadatum.path) + + expect(finder.execute!).to eq(package) + end + + it 'raises an error' do + finder = described_class.new(project, 'com/example/my-app/1.0-SNAPSHOT') + + expect { finder.execute! }.to raise_error(ActiveRecord::RecordNotFound) + end + end +end diff --git a/ee/spec/finders/packages/package_file_finder_spec.rb b/ee/spec/finders/packages/package_file_finder_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..9ee08026a64bdb64d8aa3142a9afde7bc4b9f14b --- /dev/null +++ b/ee/spec/finders/packages/package_file_finder_spec.rb @@ -0,0 +1,21 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Packages::PackageFileFinder do + let(:package) { create(:maven_package) } + let(:package_file) { package.package_files.first } + + describe '#execute!' do + it 'returns a package file' do + finder = described_class.new(package, package_file.file_name) + + expect(finder.execute!).to eq(package_file) + end + + it 'raises an error' do + finder = described_class.new(package, 'unknown.jpg') + + expect { finder.execute! }.to raise_error(ActiveRecord::RecordNotFound) + end + end +end diff --git a/ee/spec/fixtures/maven/maven-metadata.xml b/ee/spec/fixtures/maven/maven-metadata.xml new file mode 100644 index 0000000000000000000000000000000000000000..7d7549df2277cbb1044f2ed4dc860b99e023d15a --- /dev/null +++ b/ee/spec/fixtures/maven/maven-metadata.xml @@ -0,0 +1,25 @@ + + + com.mycompany.app + my-app + 1.0-SNAPSHOT + + + 20180724.124855 + 1 + + 20180724124855 + + + jar + 1.0-20180724.124855-1 + 20180724124855 + + + pom + 1.0-20180724.124855-1 + 20180724124855 + + + + diff --git a/ee/spec/fixtures/maven/my-app-1.0-20180724.124855-1.jar b/ee/spec/fixtures/maven/my-app-1.0-20180724.124855-1.jar new file mode 100644 index 0000000000000000000000000000000000000000..ea3903cf6d9f4953eb1a9678612016e29522d5b4 Binary files /dev/null and b/ee/spec/fixtures/maven/my-app-1.0-20180724.124855-1.jar differ diff --git a/ee/spec/fixtures/maven/my-app-1.0-20180724.124855-1.pom b/ee/spec/fixtures/maven/my-app-1.0-20180724.124855-1.pom new file mode 100644 index 0000000000000000000000000000000000000000..6b6015314aa280bf24dd8d47c63d4465248e5f58 --- /dev/null +++ b/ee/spec/fixtures/maven/my-app-1.0-20180724.124855-1.pom @@ -0,0 +1,34 @@ + + 4.0.0 + com.mycompany.app + my-app + jar + 1.0-SNAPSHOT + my-app + http://maven.apache.org + + + junit + junit + 3.8.1 + test + + + + + local + file:///tmp/maven + + + + + local + file:///tmp/maven + + + + 1.6 + 1.6 + + diff --git a/ee/spec/models/packages/maven_metadatum_spec.rb b/ee/spec/models/packages/maven_metadatum_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..b7c60bdd16084e08d1cfed5c438d356e84836192 --- /dev/null +++ b/ee/spec/models/packages/maven_metadatum_spec.rb @@ -0,0 +1,30 @@ +# frozen_string_literal: true +require 'rails_helper' + +RSpec.describe Packages::MavenMetadatum, type: :model do + describe 'relationships' do + it { is_expected.to belong_to(:package) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:package) } + + describe '#app_name' do + it { is_expected.to allow_value("my-app").for(:app_name) } + it { is_expected.not_to allow_value("my/app").for(:app_name) } + it { is_expected.not_to allow_value("my(app)").for(:app_name) } + end + + describe '#app_group' do + it { is_expected.to allow_value("my.domain.com").for(:app_group) } + it { is_expected.not_to allow_value("my/domain/com").for(:app_group) } + it { is_expected.not_to allow_value("my(domain)").for(:app_group) } + end + + describe '#path' do + it { is_expected.to allow_value("my/domain/com/my-app").for(:path) } + it { is_expected.to allow_value("my/domain/com/my-app/1.0-SNAPSHOT").for(:path) } + it { is_expected.not_to allow_value("my(domain)com.my-app").for(:path) } + end + end +end diff --git a/ee/spec/models/packages/package_file_spec.rb b/ee/spec/models/packages/package_file_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..a3167ef0b5ecc9f1c52bbd7feb3a84391b6e9cc4 --- /dev/null +++ b/ee/spec/models/packages/package_file_spec.rb @@ -0,0 +1,12 @@ +# frozen_string_literal: true +require 'rails_helper' + +RSpec.describe Packages::PackageFile, type: :model do + describe 'relationships' do + it { is_expected.to belong_to(:package) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:package) } + end +end diff --git a/ee/spec/models/packages/package_spec.rb b/ee/spec/models/packages/package_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..eb9ade80cb989c91fe6847c09d1204165f887788 --- /dev/null +++ b/ee/spec/models/packages/package_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true +require 'rails_helper' + +RSpec.describe Packages::Package, type: :model do + describe 'relationships' do + it { is_expected.to belong_to(:project) } + it { is_expected.to have_many(:package_files) } + end + + describe 'validations' do + it { is_expected.to validate_presence_of(:project) } + + describe '#name' do + it { is_expected.to allow_value("my/domain/com/my-app").for(:name) } + it { is_expected.to allow_value("my.app-11.07.2018").for(:name) } + it { is_expected.not_to allow_value("my(dom$$$ain)com.my-app").for(:name) } + end + end +end diff --git a/ee/spec/requests/api/maven_packages_spec.rb b/ee/spec/requests/api/maven_packages_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..e7564404f82cd3d83f3e757e06e6459d7485e43e --- /dev/null +++ b/ee/spec/requests/api/maven_packages_spec.rb @@ -0,0 +1,182 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe API::MavenPackages do + let(:user) { create(:user) } + let(:project) { create(:project, :public) } + let(:personal_access_token) { create(:personal_access_token, user: user) } + let(:jwt_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') } + let(:headers) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => jwt_token } } + let(:headers_with_token) { headers.merge('Private-Token' => personal_access_token.token) } + + before do + project.add_developer(user) + stub_licensed_features(packages: true) + end + + describe 'GET /api/v4/projects/:id/packages/maven/*path/:file_name' do + let(:package) { create(:maven_package, project: project) } + let(:maven_metadatum) { package.maven_metadatum } + let(:package_file_xml) { package.package_files.find_by(file_type: 'xml') } + + context 'a public project' do + it 'returns the file' do + download_file(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq('application/octet-stream') + end + + it 'returns sha1 of the file' do + download_file(package_file_xml.file_name + '.sha1') + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq('text/plain') + expect(response.body).to eq(package_file_xml.file_sha1) + end + end + + context 'private project' do + before do + project.update!(visibility_level: Gitlab::VisibilityLevel::PRIVATE) + end + + it 'returns the file' do + download_file_with_token(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq('application/octet-stream') + end + + it 'denies download when not enough permissions' do + project.add_guest(user) + + download_file_with_token(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(403) + end + + it 'denies download when no private token' do + download_file(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(404) + end + end + + it 'rejects request if feature is not in the license' do + stub_licensed_features(packages: false) + + download_file(package_file_xml.file_name) + + expect(response).to have_gitlab_http_status(403) + end + + def download_file(file_name, params = {}, request_headers = headers) + get api("/projects/#{project.id}/packages/maven/" \ + "#{maven_metadatum.path}/#{file_name}"), params, request_headers + end + + def download_file_with_token(file_name, params = {}, request_headers = headers_with_token) + download_file(file_name, params, request_headers) + end + end + + describe 'PUT /api/v4/projects/:id/packages/maven/*path/:file_name/authorize' do + it 'authorizes posting package with a valid token' do + authorize_upload_with_token + + expect(response).to have_gitlab_http_status(200) + expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + expect(json_response['TempPath']).not_to be_nil + end + + it 'rejects request without a valid token' do + headers_with_token['Private-Token'] = 'foo' + + authorize_upload_with_token + + expect(response).to have_gitlab_http_status(401) + end + + it 'rejects request without a valid permission' do + project.add_guest(user) + + authorize_upload_with_token + + expect(response).to have_gitlab_http_status(403) + end + + it 'rejects requests that did not go through gitlab-workhorse' do + headers.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER) + + authorize_upload_with_token + + expect(response).to have_gitlab_http_status(500) + end + + def authorize_upload(params = {}, request_headers = headers) + put api("/projects/#{project.id}/packages/maven/com/example/my-app/1.0-SNAPSHOT/maven-metadata.xml/authorize"), params, request_headers + end + + def authorize_upload_with_token(params = {}, request_headers = headers_with_token) + authorize_upload(params, request_headers) + end + end + + describe 'PUT /api/v4/projects/:id/packages/maven/*path/:file_name' do + let(:file_upload) { fixture_file_upload('ee/spec/fixtures/maven/maven-metadata.xml') } + + before do + # by configuring this path we allow to pass temp file from any path + allow(Packages::PackageFileUploader).to receive(:workhorse_upload_path).and_return('/') + end + + it 'rejects requests without a file from workhorse' do + upload_file_with_token + + expect(response).to have_gitlab_http_status(400) + end + + it 'rejects request without a token' do + upload_file + + expect(response).to have_gitlab_http_status(401) + end + + it 'rejects request if feature is not in the license' do + stub_licensed_features(packages: false) + + upload_file_with_token + + expect(response).to have_gitlab_http_status(403) + end + + context 'when params from workhorse are correct' do + let(:package) { project.packages.reload.last } + let(:package_file) { package.package_files.reload.last } + let(:params) do + { + 'file.path' => file_upload.path, + 'file.name' => file_upload.original_filename + } + end + + it 'creates package and stores package file' do + expect { upload_file_with_token(params) }.to change { project.packages.count }.by(1) + .and change { Packages::MavenMetadatum.count }.by(1) + .and change { Packages::PackageFile.count }.by(1) + + expect(response).to have_gitlab_http_status(200) + expect(package_file.file_name).to eq(file_upload.original_filename) + end + end + + def upload_file(params = {}, request_headers = headers) + put api("/projects/#{project.id}/packages/maven/com/example/my-app/1.0-SNAPSHOT/maven-metadata.xml"), params, request_headers + end + + def upload_file_with_token(params = {}, request_headers = headers_with_token) + upload_file(params, request_headers) + end + end +end diff --git a/ee/spec/services/packages/create_maven_package_service_spec.rb b/ee/spec/services/packages/create_maven_package_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..455980850602769144dd27578a0a16d7380a8ac2 --- /dev/null +++ b/ee/spec/services/packages/create_maven_package_service_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Packages::CreateMavenPackageService do + let(:project) { create(:project) } + let(:user) { create(:user) } + let(:app_name) { 'my-app'.freeze } + let(:version) { '1.0-SNAPSHOT'.freeze } + let(:path) { "my/company/app/#{app_name}" } + let(:path_with_version) { "#{path}/#{version}" } + + describe '#execute' do + context 'with version' do + let(:params) do + { + path: path_with_version, + name: path, + version: version + } + end + + it 'creates a new package with metadatum' do + package = described_class.new(project, user, params).execute + + expect(package).to be_valid + expect(package.name).to eq(path) + expect(package.version).to eq(version) + expect(package.maven_metadatum).to be_valid + expect(package.maven_metadatum.path).to eq(path_with_version) + expect(package.maven_metadatum.app_group).to eq('my.company.app') + expect(package.maven_metadatum.app_name).to eq(app_name) + expect(package.maven_metadatum.app_version).to eq(version) + end + end + + context 'without version' do + let(:params) do + { + path: path, + name: path, + version: nil + } + end + + it 'creates a new package with metadatum' do + package = described_class.new(project, user, params).execute + + expect(package).to be_valid + expect(package.name).to eq(path) + expect(package.version).to be nil + expect(package.maven_metadatum).to be_valid + expect(package.maven_metadatum.path).to eq(path) + expect(package.maven_metadatum.app_group).to eq('my.company.app') + expect(package.maven_metadatum.app_name).to eq(app_name) + expect(package.maven_metadatum.app_version).to be nil + end + end + + context 'path is missing' do + let(:params) do + { + name: path, + version: version + } + end + + it 'raises an error' do + service = described_class.new(project, user, params) + + expect { service.execute }.to raise_error(ActiveRecord::RecordInvalid) + end + end + end +end diff --git a/ee/spec/services/packages/create_package_file_service_spec.rb b/ee/spec/services/packages/create_package_file_service_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..4cd880d3f40aa41280bdd8b71b8ffa2ff690a413 --- /dev/null +++ b/ee/spec/services/packages/create_package_file_service_spec.rb @@ -0,0 +1,38 @@ +# frozen_string_literal: true +require 'spec_helper' + +describe Packages::CreatePackageFileService do + let(:package) { create(:maven_package) } + + describe '#execute' do + context 'with valid params' do + let(:params) do + { + file: Tempfile.new, + file_name: 'foo.jar' + } + end + + it 'creates a new package file' do + package_file = described_class.new(package, params).execute + + expect(package_file).to be_valid + expect(package_file.file_name).to eq('foo.jar') + end + end + + context 'file is missing' do + let(:params) do + { + file_name: 'foo.jar' + } + end + + it 'raises an error' do + service = described_class.new(package, params) + + expect { service.execute }.to raise_error(ActiveRecord::RecordInvalid) + end + end + end +end diff --git a/lib/api/api.rb b/lib/api/api.rb index 03e61cb7b31f8443348ca4833142794a44b90c80..ea42b50d80c8e66f93e72b12cc1678ec56ce4c93 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -172,6 +172,7 @@ class API < Grape::API mount ::API::License mount ::API::ProjectMirror mount ::API::ProjectPushRule + mount ::API::MavenPackages ## EE-specific API V4 endpoints END route :any, '*path' do diff --git a/spec/lib/gitlab/import_export/all_models.yml b/spec/lib/gitlab/import_export/all_models.yml index 54cf88b926110c50828ed38345f9ea9e06a2e67c..e10298cd106993b8a443d1a410eb8b26790befe3 100644 --- a/spec/lib/gitlab/import_export/all_models.yml +++ b/spec/lib/gitlab/import_export/all_models.yml @@ -348,6 +348,7 @@ project: - software_license_policies - repository_languages - project_registry +- packages award_emoji: - awardable - user