From 6c00c33403889f9209e62f451c7a614c8da82de5 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Thu, 7 Jul 2016 23:28:10 -0300 Subject: [PATCH 1/6] Add GitLab::Import::Github::Client --- lib/gitlab/import/github/client.rb | 67 +++++++++++++++++++ lib/gitlab/import/github/options.rb | 33 +++++++++ spec/lib/gitlab/import/github/options_spec.rb | 57 ++++++++++++++++ 3 files changed, 157 insertions(+) create mode 100644 lib/gitlab/import/github/client.rb create mode 100644 lib/gitlab/import/github/options.rb create mode 100644 spec/lib/gitlab/import/github/options_spec.rb diff --git a/lib/gitlab/import/github/client.rb b/lib/gitlab/import/github/client.rb new file mode 100644 index 000000000000..e286a9a22bf4 --- /dev/null +++ b/lib/gitlab/import/github/client.rb @@ -0,0 +1,67 @@ +module Gitlab + module Import + module Github + class Client + def initialize(project) + @repo = project.import_source + @repo_url = project.import_url + @credentials = project.import_data.credentials + @options = Options.new + end + + private + + attr_reader :credentials, :options, :repo, :repo_url + + SAFE_REMAINING_REQUESTS = 100 + SAFE_SLEEP_TIME = 500 + + def client + @client ||= begin + Octokit.auto_paginate = false + + Octokit::Client.new( + access_token: access_token, + api_endpoint: options.endpoint, + connection_options: { + ssl: { verify: options.verify_ssl } + } + ) + end + end + + def access_token + credentials[:user] + end + + def rate_limit + client.rate_limit! + end + + def rate_limit_exceed? + rate_limit.remaining <= SAFE_REMAINING_REQUESTS + end + + def rate_limit_sleep_time + rate_limit.resets_in + SAFE_SLEEP_TIME + end + + def request + sleep rate_limit_sleep_time if rate_limit_exceed? + + data = yield + + last_response = client.last_response + + while last_response.rels[:next] + sleep rate_limit_sleep_time if rate_limit_exceed? + last_response = last_response.rels[:next].get + data.concat(last_response.data) if last_response.data.is_a?(Array) + end + + data + end + end + end + end +end diff --git a/lib/gitlab/import/github/options.rb b/lib/gitlab/import/github/options.rb new file mode 100644 index 000000000000..dc7d8feb7c7c --- /dev/null +++ b/lib/gitlab/import/github/options.rb @@ -0,0 +1,33 @@ +module Gitlab + module Import + module Github + class Options + def endpoint + client_options[:site] + end + + def verify_ssl + options.fetch(:verify_ssl, true) + end + + private + + def client_options + @client_options ||= options.fetch(:args, options)[:client_options] + end + + def custom_options + Gitlab.config.omniauth.providers.find { |provider| provider.name == 'github' } + end + + def default_options + OmniAuth::Strategies::GitHub.default_options + end + + def options + @options ||= (custom_options || default_options).deep_symbolize_keys + end + end + end + end +end diff --git a/spec/lib/gitlab/import/github/options_spec.rb b/spec/lib/gitlab/import/github/options_spec.rb new file mode 100644 index 000000000000..2f40f9c18c94 --- /dev/null +++ b/spec/lib/gitlab/import/github/options_spec.rb @@ -0,0 +1,57 @@ +require 'spec_helper' + +describe Gitlab::Import::Github::Options, lib: true do + subject(:options) { described_class.new } + + context 'with custom options' do + let(:github) do + Settingslogic.new( + 'name' => 'github', + 'app_id' => '123456', + 'app_secret' => '123456', + 'verify_ssl' => false, + 'args' => { + 'client_options' => { + 'site' => 'https://github.mycompany.com', + 'authorize_url' => 'https://github.mycompany.com/login/oauth/authorize', + 'token_url' => 'https://github.mycompany.com/login/oauth/access_token' + } + } + ) + end + + before do + allow(Gitlab.config.omniauth).to receive(:providers).and_return([github]) + end + + describe '#endpoint' do + it 'returns custom API endpoint' do + expect(options.endpoint).to eq 'https://github.mycompany.com' + end + end + + describe '#verify_ssl' do + it 'returns custom value' do + expect(options.verify_ssl).to eq false + end + end + end + + context 'with default options' do + before do + allow(Gitlab.config.omniauth).to receive(:providers).and_return([]) + end + + describe '#endpoint' do + it 'returns GitHub API endpoint' do + expect(options.endpoint).to eq 'https://api.github.com' + end + end + + describe '#verify_ssl' do + it 'returns true' do + expect(options.verify_ssl).to eq true + end + end + end +end -- GitLab From 9af6b08fe7885d13400b6b459e47c3546d14d456 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Fri, 8 Jul 2016 14:16:04 -0300 Subject: [PATCH 2/6] Add a mapper to handle with GitHub labels --- lib/gitlab/import/github/client.rb | 4 +++ lib/gitlab/import/github/mapper/label.rb | 30 +++++++++++++++++++ .../gitlab/import/github/mapper/label_spec.rb | 30 +++++++++++++++++++ 3 files changed, 64 insertions(+) create mode 100644 lib/gitlab/import/github/mapper/label.rb create mode 100644 spec/lib/gitlab/import/github/mapper/label_spec.rb diff --git a/lib/gitlab/import/github/client.rb b/lib/gitlab/import/github/client.rb index e286a9a22bf4..228c6c5e692d 100644 --- a/lib/gitlab/import/github/client.rb +++ b/lib/gitlab/import/github/client.rb @@ -9,6 +9,10 @@ def initialize(project) @options = Options.new end + def labels + request { client.labels(repo, per_page: 100) } + end + private attr_reader :credentials, :options, :repo, :repo_url diff --git a/lib/gitlab/import/github/mapper/label.rb b/lib/gitlab/import/github/mapper/label.rb new file mode 100644 index 000000000000..6e2f4cb99a97 --- /dev/null +++ b/lib/gitlab/import/github/mapper/label.rb @@ -0,0 +1,30 @@ +module Gitlab + module Import + module Github + module Mapper + class Label + def initialize(project, client) + @project = project + @client = client + end + + def each + client.labels.each do |raw| + label = ::Label.new( + project: project, + title: raw.name, + color: "##{raw.color}" + ) + + yield(label) + end + end + + private + + attr_reader :project, :client + end + end + end + end +end diff --git a/spec/lib/gitlab/import/github/mapper/label_spec.rb b/spec/lib/gitlab/import/github/mapper/label_spec.rb new file mode 100644 index 000000000000..c02758cda442 --- /dev/null +++ b/spec/lib/gitlab/import/github/mapper/label_spec.rb @@ -0,0 +1,30 @@ +require 'spec_helper' + +describe Gitlab::Import::Github::Mapper::Label, lib: true do + let(:project) { create(:empty_project) } + let(:client) { double(labels: response) } + + let(:response) do + [ + double( + url: 'https://api.github.com/repos/octocat/Hello-World/labels/bug', + name: 'Bug', + color: 'f29513' + ) + ] + end + + subject(:mapper) { described_class.new(project, client) } + + describe '#each' do + it 'yields successively with Label' do + expect { |block| mapper.each(&block) }.to yield_successive_args(Label) + end + + it 'matches the Label attributes' do + mapper.each do |label| + expect(label).to have_attributes(project: project, name: 'Bug', color: '#f29513') + end + end + end +end -- GitLab From 15112fbe561d74ca8c687725ac17e040b0653a48 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Fri, 8 Jul 2016 14:16:28 -0300 Subject: [PATCH 3/6] Add an action to import labels from GitHub --- lib/gitlab/import/action/import_labels.rb | 19 ++++++++++++++++ .../import/action/import_labels_spec.rb | 22 +++++++++++++++++++ 2 files changed, 41 insertions(+) create mode 100644 lib/gitlab/import/action/import_labels.rb create mode 100644 spec/lib/gitlab/import/action/import_labels_spec.rb diff --git a/lib/gitlab/import/action/import_labels.rb b/lib/gitlab/import/action/import_labels.rb new file mode 100644 index 000000000000..57c6c581fad9 --- /dev/null +++ b/lib/gitlab/import/action/import_labels.rb @@ -0,0 +1,19 @@ +module Gitlab + module Import + module Action + class ImportLabels + def initialize(project, client) + @mapper = Github::Mapper::Label.new(project, client) + end + + def execute + mapper.each(&:save) + end + + private + + attr_reader :mapper + end + end + end +end diff --git a/spec/lib/gitlab/import/action/import_labels_spec.rb b/spec/lib/gitlab/import/action/import_labels_spec.rb new file mode 100644 index 000000000000..be0cb2d93dc5 --- /dev/null +++ b/spec/lib/gitlab/import/action/import_labels_spec.rb @@ -0,0 +1,22 @@ +require 'spec_helper' + +describe Gitlab::Import::Action::ImportLabels, lib: true do + let(:project) { create(:empty_project) } + let(:client) { double(labels: response) } + + let(:response) do + [ + double( + url: 'https://api.github.com/repos/octocat/Hello-World/labels/bug', + name: 'Bug', + color: 'f29513' + ) + ] + end + + subject(:action) { described_class.new(project, client) } + + describe '#execute' do + it { expect { subject.execute }.to change(Label, :count).by(1) } + end +end -- GitLab From 80f884f0355bbb97e1b65b0f212b2025a748b865 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Fri, 8 Jul 2016 15:03:53 -0300 Subject: [PATCH 4/6] Keeps track of errors when importing GitHub labels --- lib/gitlab/import/action/import_labels.rb | 15 ++++++++++++--- lib/gitlab/import/result.rb | 19 +++++++++++++++++++ .../import/action/import_labels_spec.rb | 17 +++++++++++++++-- spec/lib/gitlab/import/result_spec.rb | 17 +++++++++++++++++ 4 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 lib/gitlab/import/result.rb create mode 100644 spec/lib/gitlab/import/result_spec.rb diff --git a/lib/gitlab/import/action/import_labels.rb b/lib/gitlab/import/action/import_labels.rb index 57c6c581fad9..8e9c9788075a 100644 --- a/lib/gitlab/import/action/import_labels.rb +++ b/lib/gitlab/import/action/import_labels.rb @@ -2,17 +2,26 @@ module Gitlab module Import module Action class ImportLabels - def initialize(project, client) + def initialize(project, client, result) @mapper = Github::Mapper::Label.new(project, client) + @result = result end def execute - mapper.each(&:save) + mapper.each do |label| + next if label.save + + label.errors.full_messages.each do |error| + result.errors << "#{label.name}: #{error}" + end + end + + result end private - attr_reader :mapper + attr_reader :mapper, :result end end end diff --git a/lib/gitlab/import/result.rb b/lib/gitlab/import/result.rb new file mode 100644 index 000000000000..971a758cc603 --- /dev/null +++ b/lib/gitlab/import/result.rb @@ -0,0 +1,19 @@ +module Gitlab + module Import + class Result + attr_reader :errors + + def initialize + @errors = [] + end + + def success? + errors.blank? + end + + def failed? + !success? + end + end + end +end diff --git a/spec/lib/gitlab/import/action/import_labels_spec.rb b/spec/lib/gitlab/import/action/import_labels_spec.rb index be0cb2d93dc5..4f8691e9af49 100644 --- a/spec/lib/gitlab/import/action/import_labels_spec.rb +++ b/spec/lib/gitlab/import/action/import_labels_spec.rb @@ -3,9 +3,15 @@ describe Gitlab::Import::Action::ImportLabels, lib: true do let(:project) { create(:empty_project) } let(:client) { double(labels: response) } + let(:result) { double(errors: []) } let(:response) do [ + double( + url: 'https://api.github.com/repos/octocat/Hello-World/labels/bug', + name: 'Bug', + color: 'f29513' + ), double( url: 'https://api.github.com/repos/octocat/Hello-World/labels/bug', name: 'Bug', @@ -14,9 +20,16 @@ ] end - subject(:action) { described_class.new(project, client) } + subject(:action) { described_class.new(project, client, result) } describe '#execute' do - it { expect { subject.execute }.to change(Label, :count).by(1) } + it 'persists labels' do + expect { subject.execute }.to change(Label, :count).by(1) + end + + it 'keeps track of errors' do + result = subject.execute + expect(result.errors).to eq ['Bug: Title has already been taken'] + end end end diff --git a/spec/lib/gitlab/import/result_spec.rb b/spec/lib/gitlab/import/result_spec.rb new file mode 100644 index 000000000000..3da5f38d3404 --- /dev/null +++ b/spec/lib/gitlab/import/result_spec.rb @@ -0,0 +1,17 @@ +require 'spec_helper' + +describe Gitlab::Import::Result, lib: true do + subject(:result) { described_class.new } + + it 'success when errors is empty' do + expect(result).to be_success + expect(result).not_to be_failed + end + + it 'failed when errors is not empty' do + result.errors << 'Something goes wrong' + + expect(result).to be_failed + expect(result).not_to be_success + end +end -- GitLab From cb8e9bb6bb0cb3712b888f79d2e8e324724742ad Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Fri, 8 Jul 2016 17:04:26 -0300 Subject: [PATCH 5/6] Add a mapper to handle with GitHub milestones --- lib/gitlab/import/github/client.rb | 7 +- lib/gitlab/import/github/mapper/label.rb | 2 + lib/gitlab/import/github/mapper/milestone.rb | 37 ++++++++++ .../import/github/mapper/milestone_spec.rb | 72 +++++++++++++++++++ 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 lib/gitlab/import/github/mapper/milestone.rb create mode 100644 spec/lib/gitlab/import/github/mapper/milestone_spec.rb diff --git a/lib/gitlab/import/github/client.rb b/lib/gitlab/import/github/client.rb index 228c6c5e692d..3eaeb13c4b17 100644 --- a/lib/gitlab/import/github/client.rb +++ b/lib/gitlab/import/github/client.rb @@ -10,13 +10,18 @@ def initialize(project) end def labels - request { client.labels(repo, per_page: 100) } + request { client.labels(repo, per_page: PER_PAGE) } + end + + def milestones + client.milestones(repo, state: :all, per_page: PER_PAGE) end private attr_reader :credentials, :options, :repo, :repo_url + PER_PAGE = 100 SAFE_REMAINING_REQUESTS = 100 SAFE_SLEEP_TIME = 500 diff --git a/lib/gitlab/import/github/mapper/label.rb b/lib/gitlab/import/github/mapper/label.rb index 6e2f4cb99a97..57c0cc358105 100644 --- a/lib/gitlab/import/github/mapper/label.rb +++ b/lib/gitlab/import/github/mapper/label.rb @@ -9,6 +9,8 @@ def initialize(project, client) end def each + return enum_for(:each) unless block_given? + client.labels.each do |raw| label = ::Label.new( project: project, diff --git a/lib/gitlab/import/github/mapper/milestone.rb b/lib/gitlab/import/github/mapper/milestone.rb new file mode 100644 index 000000000000..a675defdbe92 --- /dev/null +++ b/lib/gitlab/import/github/mapper/milestone.rb @@ -0,0 +1,37 @@ +module Gitlab + module Import + module Github + module Mapper + class Milestone + def initialize(project, client) + @project = project + @client = client + end + + def each + return enum_for(:each) unless block_given? + + client.milestones.each do |raw| + milestone = ::Milestone.new( + iid: raw.number, + project: project, + title: raw.title, + description: raw.description, + due_date: raw.due_on, + state: raw.state == 'closed' ? 'closed' : 'active', + created_at: raw.created_at, + updated_at: raw.state == 'closed' ? raw.closed_at : raw.updated_at + ) + + yield(milestone) + end + end + + private + + attr_reader :project, :client + end + end + end + end +end diff --git a/spec/lib/gitlab/import/github/mapper/milestone_spec.rb b/spec/lib/gitlab/import/github/mapper/milestone_spec.rb new file mode 100644 index 000000000000..9ebcf7814244 --- /dev/null +++ b/spec/lib/gitlab/import/github/mapper/milestone_spec.rb @@ -0,0 +1,72 @@ +require 'spec_helper' + +describe Gitlab::Import::Github::Mapper::Milestone, lib: true do + let(:project) { create(:empty_project) } + let(:client) { double(milestones: response) } + + let(:response) do + [ + double( + number: 2, + state: 'open', + title: 'v2.0', + description: 'Tracking milestone for version 2.0', + created_at: '2011-04-10T20:09:31Z', + updated_at: '2014-03-03T18:58:10Z', + closed_at: nil, + due_on: '2012-10-09T23:39:01Z' + ), + double( + number: 1, + state: 'closed', + title: 'v1.0', + description: 'Tracking milestone for version 1.0', + created_at: '2011-04-10T20:09:31Z', + updated_at: '2014-03-03T18:58:10Z', + closed_at: '2012-10-09T23:39:01Z', + due_on: nil + ) + ] + end + + subject(:mapper) { described_class.new(project, client) } + + describe '#each' do + it 'yields successively with Milestone' do + expect { |block| mapper.each(&block) }.to yield_successive_args(Milestone, Milestone) + end + + it 'matches the Milestone attributes' do + milestone_opened = { + project: project, + iid: 2, + title: 'v2.0', + description: 'Tracking milestone for version 2.0', + state: 'active', + due_date: DateTime.strptime('2012-10-09T23:39:01Z'), + created_at: DateTime.strptime('2011-04-10T20:09:31Z'), + updated_at: DateTime.strptime('2014-03-03T18:58:10Z') + } + + milestone_closed = { + project: project, + iid: 1, + title: 'v1.0', + description: 'Tracking milestone for version 1.0', + state: 'closed', + due_date: nil, + created_at: DateTime.strptime('2011-04-10T20:09:31Z'), + updated_at: DateTime.strptime('2012-10-09T23:39:01Z') + } + + expected = [ + milestone_opened, + milestone_closed + ] + + mapper.each.with_index do |milestone, index| + expect(milestone).to have_attributes(expected[index]) + end + end + end +end -- GitLab From 8f3b6591ccc1dcebb0e4ae1e3cf44588b2a4afe8 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre Date: Fri, 8 Jul 2016 18:06:02 -0300 Subject: [PATCH 6/6] Extract base mapper --- lib/gitlab/import/github/mapper/base.rb | 37 +++++++++++++++++ lib/gitlab/import/github/mapper/label.rb | 31 +++++--------- lib/gitlab/import/github/mapper/milestone.rb | 41 ++++++++----------- .../gitlab/import/github/mapper/base_spec.rb | 19 +++++++++ .../gitlab/import/github/mapper/label_spec.rb | 2 +- .../import/github/mapper/milestone_spec.rb | 2 +- 6 files changed, 85 insertions(+), 47 deletions(-) create mode 100644 lib/gitlab/import/github/mapper/base.rb create mode 100644 spec/lib/gitlab/import/github/mapper/base_spec.rb diff --git a/lib/gitlab/import/github/mapper/base.rb b/lib/gitlab/import/github/mapper/base.rb new file mode 100644 index 000000000000..78e8b97eb03a --- /dev/null +++ b/lib/gitlab/import/github/mapper/base.rb @@ -0,0 +1,37 @@ +module Gitlab + module Import + module Github + module Mapper + class Base + def initialize(project, client) + @project = project + @client = client + end + + def each + return enum_for(:each) unless block_given? + + method = klass.to_s.underscore.pluralize + + client.public_send(method).each do |raw| + yield(klass.new(attributes_for(raw))) + end + end + + private + + attr_reader :project, :client + + def attributes_for + {} + end + + def klass + raise NotImplementedError, + "#{self.class} does not implement #{__method__}" + end + end + end + end + end +end diff --git a/lib/gitlab/import/github/mapper/label.rb b/lib/gitlab/import/github/mapper/label.rb index 57c0cc358105..4a56a190d18e 100644 --- a/lib/gitlab/import/github/mapper/label.rb +++ b/lib/gitlab/import/github/mapper/label.rb @@ -2,29 +2,20 @@ module Gitlab module Import module Github module Mapper - class Label - def initialize(project, client) - @project = project - @client = client - end - - def each - return enum_for(:each) unless block_given? - - client.labels.each do |raw| - label = ::Label.new( - project: project, - title: raw.name, - color: "##{raw.color}" - ) + class Label < Base + private - yield(label) - end + def attributes_for(raw) + { + project: project, + title: raw.name, + color: "##{raw.color}" + } end - private - - attr_reader :project, :client + def klass + ::Label + end end end end diff --git a/lib/gitlab/import/github/mapper/milestone.rb b/lib/gitlab/import/github/mapper/milestone.rb index a675defdbe92..4d6e8454c55d 100644 --- a/lib/gitlab/import/github/mapper/milestone.rb +++ b/lib/gitlab/import/github/mapper/milestone.rb @@ -2,34 +2,25 @@ module Gitlab module Import module Github module Mapper - class Milestone - def initialize(project, client) - @project = project - @client = client - end - - def each - return enum_for(:each) unless block_given? - - client.milestones.each do |raw| - milestone = ::Milestone.new( - iid: raw.number, - project: project, - title: raw.title, - description: raw.description, - due_date: raw.due_on, - state: raw.state == 'closed' ? 'closed' : 'active', - created_at: raw.created_at, - updated_at: raw.state == 'closed' ? raw.closed_at : raw.updated_at - ) + class Milestone < Base + private - yield(milestone) - end + def attributes_for(raw) + { + iid: raw.number, + project: project, + title: raw.title, + description: raw.description, + due_date: raw.due_on, + state: raw.state == 'closed' ? 'closed' : 'active', + created_at: raw.created_at, + updated_at: raw.state == 'closed' ? raw.closed_at : raw.updated_at + } end - private - - attr_reader :project, :client + def klass + ::Milestone + end end end end diff --git a/spec/lib/gitlab/import/github/mapper/base_spec.rb b/spec/lib/gitlab/import/github/mapper/base_spec.rb new file mode 100644 index 000000000000..f3d823b19f20 --- /dev/null +++ b/spec/lib/gitlab/import/github/mapper/base_spec.rb @@ -0,0 +1,19 @@ +require 'spec_helper' + +describe Gitlab::Import::Github::Mapper::Base, lib: true do + let(:project) { double } + let(:client) { double } + + subject(:mapper) do + klass = Class.new(described_class) + klass.new(project, client) + end + + describe '#each' do + context 'when klass is not implemented' do + it 'raises NotImplementedError' do + expect { mapper.each(&:to_s) }.to raise_error(NotImplementedError) + end + end + end +end diff --git a/spec/lib/gitlab/import/github/mapper/label_spec.rb b/spec/lib/gitlab/import/github/mapper/label_spec.rb index c02758cda442..94d810ef9f92 100644 --- a/spec/lib/gitlab/import/github/mapper/label_spec.rb +++ b/spec/lib/gitlab/import/github/mapper/label_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::Import::Github::Mapper::Label, lib: true do - let(:project) { create(:empty_project) } + let(:project) { build(:empty_project) } let(:client) { double(labels: response) } let(:response) do diff --git a/spec/lib/gitlab/import/github/mapper/milestone_spec.rb b/spec/lib/gitlab/import/github/mapper/milestone_spec.rb index 9ebcf7814244..ea580771bf79 100644 --- a/spec/lib/gitlab/import/github/mapper/milestone_spec.rb +++ b/spec/lib/gitlab/import/github/mapper/milestone_spec.rb @@ -1,7 +1,7 @@ require 'spec_helper' describe Gitlab::Import::Github::Mapper::Milestone, lib: true do - let(:project) { create(:empty_project) } + let(:project) { build(:empty_project) } let(:client) { double(milestones: response) } let(:response) do -- GitLab