From 8dee7f8c1d4752481b18b15bb6de5d225dbf73dc Mon Sep 17 00:00:00 2001 From: Cerdic Date: Tue, 20 Feb 2024 11:10:28 +0100 Subject: [PATCH 1/6] feat: API endpoint import/gitea Allow mass import fro Gitea through scripted import The endpoint is symetrical to import/github with following parameters ``` requires :personal_access_token, type: String, desc: 'Gitea personal access token' requires :repo_id, type: Integer, desc: 'Gitea repository ID' optional :new_name, type: String, desc: 'New repo name' requires :target_namespace, type: String, allow_blank: false, desc: 'Namespace or group to import repository into' optional :gitea_host_url, type: String, desc: 'Gitea Host url' optional :optional_stages, type: Hash, desc: 'Optional stages of import to be performed' optional :timeout_strategy, type: String, values: ::ProjectImportData::TIMEOUT_STRATEGIES, desc: 'Strategy for behavior on timeouts' ``` --- lib/api/api.rb | 1 + lib/api/helpers/import_gitea_helpers.rb | 62 +++++++++++++++++++++++++ lib/api/import_gitea.rb | 43 +++++++++++++++++ 3 files changed, 106 insertions(+) create mode 100644 lib/api/helpers/import_gitea_helpers.rb create mode 100644 lib/api/import_gitea.rb diff --git a/lib/api/api.rb b/lib/api/api.rb index cecb0d9001ebed..9a9309224310c7 100644 --- a/lib/api/api.rb +++ b/lib/api/api.rb @@ -267,6 +267,7 @@ def initialize(location_url) mount ::API::HelmPackages mount ::API::ImportBitbucketServer mount ::API::ImportGithub + mount ::API::ImportGitea mount ::API::Integrations mount ::API::Integrations::Slack::Events mount ::API::Integrations::Slack::Interactions diff --git a/lib/api/helpers/import_gitea_helpers.rb b/lib/api/helpers/import_gitea_helpers.rb new file mode 100644 index 00000000000000..f629db4b445911 --- /dev/null +++ b/lib/api/helpers/import_gitea_helpers.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +module API + module Helpers + module ImportGiteaHelpers + def client + @client ||= Gitlab::LegacyGithubImport::Client.new(params[:personal_access_token], **client_options) + end + + def access_params + { + github_access_token: params[:personal_access_token] + } + end + + def provider + :gitea + end + + def provider_name + :gitea + end + + def provider_url + params[:gitea_host_url] + end + + def client_options + verified_url, provider_hostname = verify_blocked_uri + + { + host: verified_url.scheme == 'https' ? provider_url : verified_url.to_s, + api_version: 'v1', + hostname: provider_hostname + } + end + + def verify_blocked_uri + @verify_blocked_uri ||= Gitlab::UrlBlocker.validate!( + provider_url, + allow_localhost: allow_local_requests?, + allow_local_network: allow_local_requests?, + schemes: %w[http https] + ) + rescue Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError => e + error!(format(s_('Specified URL "%u%" cannot be used: "%{reason}"'), u: provider_url, reason: e.message), 403) + end + + def allow_local_requests? + Gitlab::CurrentSettings.allow_local_requests_from_web_hooks_and_services? + end + + def provider_unauthorized + error!("Access denied to your #{Gitlab::ImportSources.title(provider.to_s)} account.", 401) + end + + def too_many_requests + error!('Too Many Requests', 429) + end + end + end +end diff --git a/lib/api/import_gitea.rb b/lib/api/import_gitea.rb new file mode 100644 index 00000000000000..1d55dbb9fd76dc --- /dev/null +++ b/lib/api/import_gitea.rb @@ -0,0 +1,43 @@ +# frozen_string_literal: true + +module API + class ImportGitea < ::API::ImportGithub + extend ::Gitlab::Utils::Override + before { authenticate! } + + helpers ::API::Helpers::ImportGiteaHelpers + + desc 'Import a Gitea project' do + detail 'This feature is experimental' + success code: 201, model: ::ProjectEntity + failure [ + { code: 400, message: 'Bad request' }, + { code: 401, message: 'Unauthorized' }, + { code: 403, message: 'Forbidden' }, + { code: 422, message: 'Unprocessable entity' }, + { code: 503, message: 'Service unavailable' } + ] + tags ['project_import_gitea'] + end + params do + requires :personal_access_token, type: String, desc: 'Gitea personal access token' + requires :repo_id, type: Integer, desc: 'Gitea repository ID' + optional :new_name, type: String, desc: 'New repo name' + requires :target_namespace, type: String, allow_blank: false, desc: 'Namespace or group to import repository into' + optional :gitea_host_url, type: String, desc: 'Gitea Host url' + optional :optional_stages, type: Hash, desc: 'Optional stages of import to be performed' + optional :timeout_strategy, type: String, values: ::ProjectImportData::TIMEOUT_STRATEGIES, + desc: 'Strategy for behavior on timeouts' + end + post 'import/gitea' do + result = Import::GithubService.new(client, current_user, params).execute(access_params, provider) + + if result[:status] == :success + present ProjectSerializer.new.represent(result[:project]) + else + status result[:http_status] + { errors: result[:message] } + end + end + end +end -- GitLab From 44de99c74eedf589430fd5e633d990755c0b0420 Mon Sep 17 00:00:00 2001 From: Cerdic Date: Thu, 22 Feb 2024 10:33:16 +0100 Subject: [PATCH 2/6] fix: use an existing error message as in gitea_controller --- lib/api/helpers/import_gitea_helpers.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/api/helpers/import_gitea_helpers.rb b/lib/api/helpers/import_gitea_helpers.rb index f629db4b445911..014c4611cbd45a 100644 --- a/lib/api/helpers/import_gitea_helpers.rb +++ b/lib/api/helpers/import_gitea_helpers.rb @@ -43,7 +43,7 @@ def verify_blocked_uri schemes: %w[http https] ) rescue Gitlab::HTTP_V2::UrlBlocker::BlockedUrlError => e - error!(format(s_('Specified URL "%u%" cannot be used: "%{reason}"'), u: provider_url, reason: e.message), 403) + error!(format(s_('Specified URL cannot be used: "%{reason}"'), reason: e.message), 403) end def allow_local_requests? -- GitLab From 9f3cca743ca4644e818a4efb88b2e03aa1d276f0 Mon Sep 17 00:00:00 2001 From: Cerdic Date: Mon, 4 Mar 2024 14:33:30 +0100 Subject: [PATCH 3/6] Documentation for the /import/gitea API endpoint --- doc/api/import.md | 38 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/doc/api/import.md b/doc/api/import.md index 261340e6924ec6..4bb34f7940a9ab 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -223,6 +223,44 @@ curl --request POST \ }' ``` +## Import repository from Gitea Server + +Import your projects from Gitea Server to GitLab using the API. + +Prerequisites: + +- [Prerequisites for Gitea importer](../user/project/import/gitea.md#prerequisites). +- The namespace set in `target_namespace` must exist. +- The namespace can be your user namespace or an existing group that you have at least the Maintainer role for. + +```plaintext +POST /import/gitea +``` + +| Attribute | Type | Required | Description | +|----------------------------|---------|----------|------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `personal_access_token` | string | yes | Gitea personal access token | +| `repo_id` | integer | yes | Gitea repository ID | +| `new_name` | string | no | New repository name | +| `target_namespace` | string | yes | Namespace to import repository into. Supports subgroups like `/namespace/subgroup`. Must not be blank | +| `gitea_host_url` | string | no | Gitea Host url | +| `optional_stages` | object | no | [Additional items to import](../user/project/import/github.md#select-additional-items-to-import). | +| `timeout_strategy` | string | no | Strategy for handling import timeouts. Valid values are `optimistic` (continue to next stage of import) or `pessimistic` (fail immediately). Defaults to `pessimistic`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/422979) in GitLab 16.5. | + +```shell +curl --request POST \ + --url "https://gitlab.example.com/api/v4/import/gitea" \ + --header "content-type: application/json" \ + --header "Authorization: Bearer " \ + --data '{ + "personal_access_token": "aBc123abC12aBc123abC12abC123+_A/c123", + "repo_id": "12345", + "target_namespace": "group/subgroup", + "new_name": "NEW-NAME", + "gitea_host_url": "https://mygitea.example.com" +}' +``` + ## Related topics - [Group migration by direct transfer API](bulk_imports.md). -- GitLab From e359674ef5814e28410cb1ff2cefd57d1bffd18a Mon Sep 17 00:00:00 2001 From: Cerdic Date: Wed, 20 Mar 2024 09:49:38 +0100 Subject: [PATCH 4/6] Append an history item in the documentation for the /import/gitea API endpoint --- doc/api/import.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/api/import.md b/doc/api/import.md index 4bb34f7940a9ab..afebf0f2267bab 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -225,6 +225,8 @@ curl --request POST \ ## Import repository from Gitea Server +> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/442717) in GitLab 16.11. + Import your projects from Gitea Server to GitLab using the API. Prerequisites: -- GitLab From 3c0a0ef02442fef380cfb70be890f6576c473622 Mon Sep 17 00:00:00 2001 From: Cerdic Date: Thu, 21 Mar 2024 10:04:35 +0100 Subject: [PATCH 5/6] Corrections on import API endpoint documentation (typo and copy/paste mistake) --- doc/api/import.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/api/import.md b/doc/api/import.md index afebf0f2267bab..bf2c0c82173de9 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -245,9 +245,9 @@ POST /import/gitea | `repo_id` | integer | yes | Gitea repository ID | | `new_name` | string | no | New repository name | | `target_namespace` | string | yes | Namespace to import repository into. Supports subgroups like `/namespace/subgroup`. Must not be blank | -| `gitea_host_url` | string | no | Gitea Host url | +| `gitea_host_url` | string | no | Gitea host URL | | `optional_stages` | object | no | [Additional items to import](../user/project/import/github.md#select-additional-items-to-import). | -| `timeout_strategy` | string | no | Strategy for handling import timeouts. Valid values are `optimistic` (continue to next stage of import) or `pessimistic` (fail immediately). Defaults to `pessimistic`. [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/422979) in GitLab 16.5. | +| `timeout_strategy` | string | no | Strategy for handling import timeouts. Valid values are `optimistic` (continue to next stage of import) or `pessimistic` (fail immediately). Defaults to `pessimistic`. | ```shell curl --request POST \ -- GitLab From 0f2dcbe651d5408d590e3bb31cbf646815a29fb7 Mon Sep 17 00:00:00 2001 From: Evan Read Date: Fri, 22 Mar 2024 09:14:52 +0000 Subject: [PATCH 6/6] Change the title in the documentation entry to be more generic --- doc/api/import.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/api/import.md b/doc/api/import.md index bf2c0c82173de9..7a045ee8b30702 100644 --- a/doc/api/import.md +++ b/doc/api/import.md @@ -223,7 +223,7 @@ curl --request POST \ }' ``` -## Import repository from Gitea Server +## Import a repository from Gitea > - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/442717) in GitLab 16.11. -- GitLab