diff --git a/GITLAB_WORKHORSE_VERSION b/GITLAB_WORKHORSE_VERSION index ba624dacf9ace2d1a5ceae2a065fd8e527403529..72963fb08c2187b1f7d9c6110e4dcc4749982402 100644 --- a/GITLAB_WORKHORSE_VERSION +++ b/GITLAB_WORKHORSE_VERSION @@ -1 +1 @@ -8.20.0 +8.21.0 diff --git a/changelogs/unreleased/georgekoltsov-group-import-api.yml b/changelogs/unreleased/georgekoltsov-group-import-api.yml new file mode 100644 index 0000000000000000000000000000000000000000..b6c497afdaf3886e3d157bcf8607ea966c2932c1 --- /dev/null +++ b/changelogs/unreleased/georgekoltsov-group-import-api.yml @@ -0,0 +1,5 @@ +--- +title: Add Group Import API endpoint & update Group Import/Export documentation +merge_request: 20353 +author: +type: added diff --git a/doc/api/group_import_export.md b/doc/api/group_import_export.md new file mode 100644 index 0000000000000000000000000000000000000000..c97a753d298e5b9aff4c441e99446bd452579a27 --- /dev/null +++ b/doc/api/group_import_export.md @@ -0,0 +1,83 @@ +# Group Import/Export API + +> Introduced in GitLab 12.8 as an experimental feature. May change in future releases. + +Group Import/Export functionality allows to export group structure and import it at a new location. +Used in combination with [Project Import/Export](project_import_export.md) it allows you to preserve connections with group level relations +(e.g. a connection between a project issue and group epic). + +Group Export includes: + +1. Group Milestones +1. Group Boards +1. Group Labels +1. Group Badges +1. Group Members +1. Sub-groups (each sub-group includes all data above) + +## Schedule new export + +Start a new group export. + +```text +POST /groups/:id/export +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ---------------------------------------- | +| `id` | integer/string | yes | ID of the groupd owned by the authenticated user | + +```shell +curl --request POST --header "PRIVATE-TOKEN: " https://gitlab.example.com/api/v4/groups/1/export +``` + +```json +{ + "message": "202 Accepted" +} +``` + +## Export download + +Download the finished export. + +```text +GET /groups/:id/export/download +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ---------------------------------------- | +| `id` | integer/string | yes | ID of the group owned by the authenticated user | + +```shell +curl --header "PRIVATE-TOKEN: " --remote-header-name --remote-name https://gitlab.example.com/api/v4/groups/1/export/download +``` + +```shell +ls *export.tar.gz +2020-12-05_22-11-148_namespace_export.tar.gz +``` + +Time spent on exporting a group may vary depending on a size of the group. Export download endpoint will return exported archive once it is available. 404 is returned otherwise. + +## Import a file + +```text +POST /groups/import +``` + +| Attribute | Type | Required | Description | +| --------- | -------------- | -------- | ---------------------------------------- | +| `name` | string | yes | The name of the group to be imported | +| `path` | string | yes | Name and path for new group | +| `file` | string | yes | The file to be uploaded | +| `parent_id` | integer | no | ID of a parent group that the group will be imported into. Defaults to the current user's namespace if not provided. | + +To upload a file from your file system, use the `--form` argument. This causes +cURL to post data using the header `Content-Type: multipart/form-data`. +The `file=` parameter must point to a file on your file system and be preceded +by `@`. For example: + +```shell +curl --request POST --header "PRIVATE-TOKEN: " --form "name=imported-group" --form "path=imported-group" --form "file=@/path/to/file" https://gitlab.example.com/api/v4/groups/import +``` diff --git a/doc/api/groups.md b/doc/api/groups.md index 0a2446a8be92d7c201df3752f9776d06579fb29a..3d2ac8c1e1847e1f908250e8646928b06d1abe32 100644 --- a/doc/api/groups.md +++ b/doc/api/groups.md @@ -875,3 +875,7 @@ And to switch pages add: ## Group badges Read more in the [Group Badges](group_badges.md) documentation. + +## Group Import/Export + +Read more in the [Group Import/Export](group_import_export.md) documentation. diff --git a/lib/api/api.rb b/lib/api/api.rb index e75fd7e88a18cbecdec5f2bca2687212efba1728..9a1e0e3f8e92a22746a8787bd3b81b4f1143945d 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -130,6 +130,7 @@ class API < Grape::API mount ::API::GroupBoards mount ::API::GroupClusters mount ::API::GroupExport + mount ::API::GroupImport mount ::API::GroupLabels mount ::API::GroupMilestones mount ::API::Groups diff --git a/lib/api/group_import.rb b/lib/api/group_import.rb new file mode 100644 index 0000000000000000000000000000000000000000..de7fdc2724335b0bcce715691c89871e55aba05e --- /dev/null +++ b/lib/api/group_import.rb @@ -0,0 +1,77 @@ +# frozen_string_literal: true + +module API + class GroupImport < Grape::API + MAXIMUM_FILE_SIZE = 50.megabytes.freeze + + helpers do + def authorize_create_group! + parent_group = find_group!(params[:parent_id]) if params[:parent_id].present? + + if parent_group + authorize! :create_subgroup, parent_group + else + authorize! :create_group + end + end + end + + resource :groups, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do + desc 'Workhorse authorize the group import upload' do + detail 'This feature was introduced in GitLab 12.8' + end + post 'import/authorize' do + require_gitlab_workhorse! + + Gitlab::Workhorse.verify_api_request!(headers) + + status 200 + content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE + + ImportExportUploader.workhorse_authorize(has_length: false, maximum_size: MAXIMUM_FILE_SIZE) + end + + desc 'Create a new group import' do + detail 'This feature was introduced in GitLab 12.8' + success Entities::Group + end + params do + requires :path, type: String, desc: 'Group path' + requires :name, type: String, desc: 'Group name' + optional :parent_id, type: Integer, desc: "The ID of the parent group that the group will be imported into. Defaults to the current user's namespace." + optional 'file.path', type: String, desc: 'Path to locally stored body (generated by Workhorse)' + optional 'file.name', type: String, desc: 'Real filename as send in Content-Disposition (generated by Workhorse)' + optional 'file.type', type: String, desc: 'Real content type as send in Content-Type (generated by Workhorse)' + optional 'file.size', type: Integer, desc: 'Real size of file (generated by Workhorse)' + optional 'file.md5', type: String, desc: 'MD5 checksum of the file (generated by Workhorse)' + optional 'file.sha1', type: String, desc: 'SHA1 checksum of the file (generated by Workhorse)' + optional 'file.sha256', type: String, desc: 'SHA256 checksum of the file (generated by Workhorse)' + end + post 'import' do + authorize_create_group! + require_gitlab_workhorse! + + uploaded_file = UploadedFile.from_params(params, :file, ImportExportUploader.workhorse_local_upload_path) + + bad_request!('Unable to process group import file') unless uploaded_file + + group_params = { + path: params[:path], + name: params[:name], + parent_id: params[:parent_id], + import_export_upload: ImportExportUpload.new(import_file: uploaded_file) + } + + group = ::Groups::CreateService.new(current_user, group_params).execute + + if group.persisted? + GroupImportWorker.perform_async(current_user.id, group.id) + + accepted! + else + render_api_error!("Failed to save group #{group.errors.messages}", 400) + end + end + end + end +end diff --git a/lib/api/helpers/file_upload_helpers.rb b/lib/api/helpers/file_upload_helpers.rb new file mode 100644 index 0000000000000000000000000000000000000000..c5fb291a2b7ce98024459d4ab1b3ee6e3d290d99 --- /dev/null +++ b/lib/api/helpers/file_upload_helpers.rb @@ -0,0 +1,15 @@ +# frozen_string_literal: true + +module API + module Helpers + module FileUploadHelpers + def file_is_valid? + params[:file] && params[:file]['tempfile'].respond_to?(:read) + end + + def validate_file! + render_api_error!('Uploaded file is invalid', 400) unless file_is_valid? + end + end + end +end diff --git a/lib/api/project_import.rb b/lib/api/project_import.rb index 7e0bd299761bdebdefe625658469070e5b9294b4..ea793a09f6cf5773597f63e431446f0aaae0eb79 100644 --- a/lib/api/project_import.rb +++ b/lib/api/project_import.rb @@ -5,20 +5,13 @@ class ProjectImport < Grape::API include PaginationParams helpers Helpers::ProjectsHelpers + helpers Helpers::FileUploadHelpers helpers do def import_params declared_params(include_missing: false) end - def file_is_valid? - import_params[:file] && import_params[:file]['tempfile'].respond_to?(:read) - end - - def validate_file! - render_api_error!('The file is invalid', 400) unless file_is_valid? - end - def throttled?(key, scope) rate_limiter.throttled?(key, scope: scope) end diff --git a/spec/requests/api/group_import_spec.rb b/spec/requests/api/group_import_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..016ed6ff49131bd446f17f8dd3dba8dfe81872a7 --- /dev/null +++ b/spec/requests/api/group_import_spec.rb @@ -0,0 +1,268 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe API::GroupImport do + include WorkhorseHelpers + + let_it_be(:user) { create(:user) } + let_it_be(:group) { create(:group) } + let(:path) { '/groups/import' } + let(:file) { File.join('spec', 'fixtures', 'group_export.tar.gz') } + let(:export_path) { "#{Dir.tmpdir}/group_export_spec" } + let(:workhorse_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') } + let(:workhorse_header) { { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token } } + + before do + allow_next_instance_of(Gitlab::ImportExport) do |import_export| + expect(import_export).to receive(:storage_path).and_return(export_path) + end + + stub_uploads_object_storage(ImportExportUploader) + end + + after do + FileUtils.rm_rf(export_path, secure: true) + end + + describe 'POST /groups/import' do + let(:file_upload) { fixture_file_upload(file) } + let(:params) do + { + path: 'test-import-group', + name: 'test-import-group', + file: fixture_file_upload(file) + } + end + + subject { post api('/groups/import', user), params: params, headers: workhorse_header } + + shared_examples 'when all params are correct' do + context 'when user is authorized to create new group' do + it 'creates new group and accepts request' do + subject + + expect(response).to have_gitlab_http_status(202) + end + + context 'when importing to a parent group' do + before do + group.add_owner(user) + end + + it 'creates new group and accepts request' do + params[:parent_id] = group.id + + subject + + expect(response).to have_gitlab_http_status(202) + expect(group.children.count).to eq(1) + end + + context 'when parent group is invalid' do + it 'returns 404 and does not create new group' do + params[:parent_id] = 99999 + + expect { subject }.not_to change { Group.count } + + expect(response).to have_gitlab_http_status(404) + expect(json_response['message']).to eq('404 Group Not Found') + end + + context 'when user is not an owner of parent group' do + it 'returns 403 Forbidden HTTP status' do + params[:parent_id] = create(:group).id + + subject + + expect(response).to have_gitlab_http_status(403) + expect(json_response['message']).to eq('403 Forbidden') + end + end + end + end + + context 'when group creation failed' do + before do + allow_next_instance_of(Group) do |group| + allow(group).to receive(:persisted?).and_return(false) + end + end + + it 'returns 400 HTTP status' do + subject + + expect(response).to have_gitlab_http_status(400) + end + end + end + + context 'when user is not authorized to create new group' do + let(:user) { create(:user, can_create_group: false) } + + it 'forbids the request' do + subject + + expect(response).to have_gitlab_http_status(403) + end + end + end + + shared_examples 'when some params are missing' do + context 'when required params are missing' do + shared_examples 'missing parameter' do |params, error_message| + it 'returns 400 HTTP status' do + params[:file] = file_upload + + expect do + post api('/groups/import', user), params: params, headers: workhorse_header + end.not_to change { Group.count }.from(1) + + expect(response).to have_gitlab_http_status(400) + expect(json_response['error']).to eq(error_message) + end + end + + include_examples 'missing parameter', { name: 'test' }, 'path is missing' + include_examples 'missing parameter', { path: 'test' }, 'name is missing' + end + end + + context 'with object storage disabled' do + before do + stub_uploads_object_storage(ImportExportUploader, enabled: false) + end + + context 'without a file from workhorse' do + it 'rejects the request' do + subject + + expect(response).to have_gitlab_http_status(:bad_request) + end + end + + context 'without a workhorse header' do + it 'rejects request without a workhorse header' do + post api('/groups/import', user), params: params + + expect(response).to have_gitlab_http_status(:forbidden) + end + end + + context 'when params from workhorse are correct' do + let(:params) do + { + path: 'test-import-group', + name: 'test-import-group', + 'file.path' => file_upload.path, + 'file.name' => file_upload.original_filename + } + end + + include_examples 'when all params are correct' + include_examples 'when some params are missing' + end + + it "doesn't attempt to migrate file to object storage" do + expect(ObjectStorage::BackgroundMoveWorker).not_to receive(:perform_async) + + subject + end + end + + context 'with object storage enabled' do + before do + stub_uploads_object_storage(ImportExportUploader, enabled: true) + + allow(ImportExportUploader).to receive(:workhorse_upload_path).and_return('/') + end + + context 'with direct upload enabled' do + let(:file_name) { 'group_export.tar.gz' } + let!(:fog_connection) do + stub_uploads_object_storage(ImportExportUploader, direct_upload: true) + end + let(:tmp_object) do + fog_connection.directories.new(key: 'uploads').files.create( + key: "tmp/uploads/#{file_name}", + body: file_upload + ) + end + let(:fog_file) { fog_to_uploaded_file(tmp_object) } + let(:params) do + { + path: 'test-import-group', + name: 'test-import-group', + file: fog_file, + 'file.remote_id' => file_name, + 'file.size' => fog_file.size + } + end + + it 'accepts the request and stores the file' do + expect { subject }.to change { Group.count }.by(1) + + expect(response).to have_gitlab_http_status(:accepted) + end + + include_examples 'when all params are correct' + include_examples 'when some params are missing' + end + end + end + + describe 'POST /groups/import/authorize' do + subject { post api('/groups/import/authorize', user), headers: workhorse_header } + + it 'authorizes importing group with workhorse header' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + end + + it 'rejects requests that bypassed gitlab-workhorse' do + workhorse_header.delete(Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER) + + subject + + expect(response).to have_gitlab_http_status(:forbidden) + end + + context 'when using remote storage' do + context 'when direct upload is enabled' do + before do + stub_uploads_object_storage(ImportExportUploader, enabled: true, direct_upload: true) + end + + it 'responds with status 200, location of file remote store and object details' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + expect(json_response).not_to have_key('TempPath') + expect(json_response['RemoteObject']).to have_key('ID') + expect(json_response['RemoteObject']).to have_key('GetURL') + expect(json_response['RemoteObject']).to have_key('StoreURL') + expect(json_response['RemoteObject']).to have_key('DeleteURL') + expect(json_response['RemoteObject']).to have_key('MultipartUpload') + end + end + + context 'when direct upload is disabled' do + before do + stub_uploads_object_storage(ImportExportUploader, enabled: true, direct_upload: false) + end + + it 'handles as a local file' do + subject + + expect(response).to have_gitlab_http_status(:ok) + expect(response.content_type.to_s).to eq(Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE) + expect(json_response['TempPath']).to eq(ImportExportUploader.workhorse_local_upload_path) + expect(json_response['RemoteObject']).to be_nil + end + end + end + end +end