diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e1307a2ff8ae59ccd0fef1e216b8837fb333e389..cc313a7fbe88ff75229d49801bde6f96db3f2cfc 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -163,6 +163,11 @@ populate_review_db: - docker rm insights-review-db-populate retry: 2 when: manual + artifacts: + expire_in: 31d + paths: + - tmp/profiles + when: always populate_prod_db: <<: *clean diff --git a/Gemfile b/Gemfile index 1d336c290bbd2122e7eae97b83a6aebeced48e14..1547a2a981c0883114f69c428df5b6cef8386f59 100644 --- a/Gemfile +++ b/Gemfile @@ -37,6 +37,10 @@ gem 'chart-js-rails' gem 'chroma' gem 'semantic' gem 'descriptive_statistics' +gem 'concurrent-ruby' +gem 'get_process_mem' +gem 'memory_profiler' +gem 'yajl-ruby', require: 'yajl/json_gem' group :development, :test do gem 'rspec' diff --git a/Gemfile.lock b/Gemfile.lock index 5b28899848aff04b3b1e50eba52329515e1608ac..75dbfe28b54060fc26c4178bdcc37f6422ba2f7f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -67,6 +67,8 @@ GEM railties (>= 3.0.0) ffi (1.9.23) forgery (0.7.0) + get_process_mem (0.2.5) + ffi (~> 1.0) gitlab (4.3.0) httparty terminal-table @@ -107,6 +109,7 @@ GEM nokogiri (>= 1.5.9) mail (2.7.0) mini_mime (>= 0.1.1) + memory_profiler (0.9.14) mini_mime (1.0.0) mini_portile2 (2.3.0) minitest (5.11.3) @@ -221,6 +224,7 @@ GEM binding_of_caller (>= 0.7.2) railties (>= 4.0) sprockets-rails (>= 2.0, < 4.0) + yajl-ruby (1.4.1) PLATFORMS ruby @@ -230,16 +234,19 @@ DEPENDENCIES chart-js-rails chroma coffee-rails (~> 4.1.0) + concurrent-ruby database_cleaner descriptive_statistics factory_bot_rails forgery + get_process_mem gitlab graphiql-rails graphql haml-rails (~> 1.0) jbuilder (~> 2.0) jquery-rails + memory_profiler pg rails (= 4.2.10) rspec @@ -254,6 +261,7 @@ DEPENDENCIES timecop uglifier (>= 1.3.0) web-console (~> 2.0) + yajl-ruby BUNDLED WITH 1.17.3 diff --git a/README.md b/README.md index d5b22ab77974ecbb2b1b592f833575afe2e7bd54..47ffa5c59a62cc68b57b986ad88a787ad5019d8a 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ For more info see [adding data pages](docs/adding_data.md) 1. Create an access token on GitLab.com to use for retrieving data -Run the limited populate step to add some data: +Run the limited populate step to add some data (500 issues and 500 MRs per project): ``` bundle exec rake "gitlab_insights:populate_limited[personal_access_token]" RAILS_ENV=development @@ -61,11 +61,17 @@ bundle exec rake "gitlab_insights:populate[personal_access_token]" RAILS_ENV=dev If adding projects for `dev` include a second access token for that instance: - ``` bundle exec rake "gitlab_insights:populate[personal_access_token, dev_access_token]" RAILS_ENV=development ``` +You can also limit the number of issues and MRs to import per project with the `LIMIT` +environment variable: + +``` +bundle exec rake "gitlab_insights:populate[personal_access_token]" RAILS_ENV=development LIMIT=150 +``` + ### Start the app ``` diff --git a/app/models/project.rb b/app/models/project.rb index 3b844e050b14d82ca4f0e58dd4ddd52fd17c8ef9..27b03b86ea8c29dfd875362b6cdc064bd3fbe274 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -1,12 +1,16 @@ -class Project < ActiveRecord::Base +# frozen_string_literal: true + +require_dependency 'group' + +class Project < ActiveRecord::Base belongs_to :group - has_many :revisions + has_many :revisions, dependent: :delete_all - has_many :issues - has_many :labels - has_many :merge_requests - has_many :milestones - has_many :pipelines + has_many :issues, dependent: :delete_all + has_many :labels, dependent: :delete_all + has_many :merge_requests, dependent: :delete_all + has_many :milestones, dependent: :delete_all + has_many :pipelines, dependent: :delete_all validates :path, uniqueness: true diff --git a/lib/gitlab_insights/api/async.rb b/lib/gitlab_insights/api/async.rb new file mode 100644 index 0000000000000000000000000000000000000000..01052853800d24906a660033cfb960949e4d44ad --- /dev/null +++ b/lib/gitlab_insights/api/async.rb @@ -0,0 +1,111 @@ +# frozen_string_literal: true + +require 'concurrent' + +module GitlabInsights + module Api + module Async + include Retryable + + DEFAULT_OPTIONS = { + per_page: 100 + }.freeze + DEFAULT_PAGES_BATCH_SIZE = 5 + MAX_PAGES_TO_FETCH_SIMULTANEOUSLY = 50 + ApiRequestFailedError = Class.new(StandardError) + + # This method fetches resources of type `resource_type` asynchronously for + # `project_path` from the API up to the asked `limit` starting from page `page_offset`. + def retrieve_resources(limit: 0, page_offset: 0, &block) + limit = [0, limit.to_i].max + pages_range = pages_range(page_offset, pages_to_fetch_for_limit(limit)) + + results = [] + remaining_async_pages = async_retrieve_pages(pages_range) + last_async_page = remaining_async_pages.last + + loop do + available_pages, remaining_async_pages = remaining_async_pages.partition(&:fulfilled?) + + if remaining_async_pages.any?(&:rejected?) + raise ApiRequestFailedError, "💥 The following queries failed:\n#{remaining_async_pages.select(&:rejected?)}" + end + + if available_pages.any? + available_pages.each do |page| + page.value.each do |resource| + hash_resource = resource.to_h + yield hash_resource + results << hash_resource['id'] + + break unless more_results_asked?(results, limit) + end + end + end + + break if remaining_async_pages.empty? + sleep 0.1 + end + + if more_results_asked?(results, limit) && api_has_more_results?(last_async_page) + results += retrieve_resources(limit: limit - results.size, page_offset: pages_range.last, &block) + end + + results + end + + private + + # Don't request more than 50 pages in parallel + def pages_to_fetch_for_limit(limit) + if limit.zero? + DEFAULT_PAGES_BATCH_SIZE + else + [MAX_PAGES_TO_FETCH_SIMULTANEOUSLY, (limit.to_f / DEFAULT_OPTIONS[:per_page]).ceil].min + end + end + + def pages_range(page_offset, pages_to_fetch) + start_page = page_offset + 1 + end_page = page_offset + pages_to_fetch + + (start_page..end_page) + end + + def more_results_asked?(results, limit) + limit.zero? || results.size < limit + end + + def api_has_more_results?(last_async_page) + last_async_page && last_async_page.fulfilled? && last_async_page.value.has_next_page? + end + + def async_retrieve_pages(pages_range) + pages_range.each_with_object([]) do |page, memo| + request_options = DEFAULT_OPTIONS.merge(page: page) + log_async_retrieve_page(request_options) + memo << async_retrieve_page(request_options) + end + end + + # This method fetches resources asynchronously from the API with Concurrent::Future + def async_retrieve_page(request_options) + Concurrent::Future.execute(dup_on_deref: true) { sync_api_call(request_options) }.tap do |future| + add_observer(future) + end + end + + def sync_api_call(request_options) + raise NotImplementedError + end + + def log_async_retrieve_page(request_options) + # Do nothing by default + end + + def add_observer(future) + # Do nothing by default + end + end + end +end diff --git a/lib/gitlab_insights/api/client.rb b/lib/gitlab_insights/api/client.rb index 644ae9152c540377e51179d77704dfe08ade5b46..34a90a648bc17e4fc7a52ae6720d41c5f3209e35 100644 --- a/lib/gitlab_insights/api/client.rb +++ b/lib/gitlab_insights/api/client.rb @@ -1,39 +1,12 @@ +# frozen_string_literal: true + require 'gitlab' module GitlabInsights module Api - class Client - attr_reader :client - - def initialize(endpoint, token = nil) - @client = Gitlab.client(endpoint: endpoint, private_token: token) - end - - def retrieve_resources(resource_type, path, limit = nil, options = default_options) - [].tap do |results| - # HACK because API labels doesn't accept options - print '.' - - if resource_type != :labels - first_response = @client.send(resource_type, path, options) - else - first_response = @client.send(resource_type, path) - end - - first_response.auto_paginate do |response| - print '.' - results << response - break if limit && results.length >= limit - end - end.map(&:to_hash) - end - - private - - def default_options - { - per_page: 100 - } + class Client < SimpleDelegator + def initialize(endpoint:, token: nil) + super(Gitlab.client(endpoint: endpoint, private_token: token)) end end end diff --git a/lib/gitlab_insights/api/project_resources.rb b/lib/gitlab_insights/api/project_resources.rb new file mode 100644 index 0000000000000000000000000000000000000000..af571f58e068d5399094741816acacb6e52681eb --- /dev/null +++ b/lib/gitlab_insights/api/project_resources.rb @@ -0,0 +1,56 @@ +# frozen_string_literal: true + +module GitlabInsights + module Api + class ProjectResources + include Async + + class Observer + def initialize(project_path) + @project_path = project_path + @logger = ProjectLogger.new(project_path) + end + + def update(time, value, reason) + if reason.nil? + logger.log "===> Retrieved #{value.size} resources" + else + logger.log "===> Future failed due to #{reason}" + end + rescue => e + puts e + end + + private + + attr_reader :project_path, :logger + end + + def initialize(project_path:, resource_type:, api_client:) + @api_client = api_client + @project_path = project_path + @resource_type = resource_type + @logger = ProjectLogger.new(project_path) + end + + private + + attr_reader :project_path, :resource_type, :api_client, :logger + + + def sync_api_call(request_options) + execute_with_retry(Net::ReadTimeout) do + @api_client.public_send(resource_type, project_path, request_options) + end + end + + def log_async_retrieve_page(request_options) + logger.log "===> Requesting #{resource_type} asynchronously with `#{request_options}`" + end + + def add_observer(future) + future.add_observer(Observer.new(project_path)) + end + end + end +end diff --git a/lib/gitlab_insights/api/retryable.rb b/lib/gitlab_insights/api/retryable.rb new file mode 100644 index 0000000000000000000000000000000000000000..5c767d2ad8f9649053c613b1a84ae3b8440f34a9 --- /dev/null +++ b/lib/gitlab_insights/api/retryable.rb @@ -0,0 +1,31 @@ +# frozen_string_literal: true + +module GitlabInsights + module Api + module Retryable + MAX_RETRIES = 3 + + def execute_with_retry(exception_type = StandardError) + tries = 0 + + until maximum_retries_reached?(tries) + begin + tries += 1 + result = yield + break + rescue exception_type # rubocop:disable Naming/RescuedExceptionsVariableName + raise if maximum_retries_reached?(tries) + end + end + + result + end + + private + + def maximum_retries_reached?(tries) + tries == MAX_RETRIES + end + end + end +end diff --git a/lib/gitlab_insights/db_populator.rb b/lib/gitlab_insights/db_populator.rb index 180e9f1daaba869551d9c5cb6ac48e15280895af..b3e3bfd7d7da8f15af36ea41cf3d8faf49b7a872 100644 --- a/lib/gitlab_insights/db_populator.rb +++ b/lib/gitlab_insights/db_populator.rb @@ -1,81 +1,32 @@ +# frozen_string_literal: true + module GitlabInsights class DbPopulator - attr_reader :client, :dev_client - - def initialize(client, dev_client = nil) - @client, @dev_client = client, dev_client + def initialize(com_client, dev_client = nil) + @com_client = com_client + @dev_client = dev_client if dev_token_necessary? && dev_client.nil? - raise ArgumentError.new('Dev token required for dev projects') - end - end - - def execute!(limit = nil) - Group.find_each do |group| - group.projects.find_each do |project| - project_log(project, 'Retrieving remote project resources', true) - - resources = { - issues: retrieve_resources(:issues, project, limit), - merge_requests: retrieve_resources(:merge_requests, project, limit) - } - - puts - project_log(project, 'Retrieved remote project resources') - project_log(project, 'Clearing local project resources', true) - - Issue.delete_all(project: project) - MergeRequest.delete_all(project: project) - - project_log(project, 'Cleared local project resources') - - add_resources_to_project(project, resources) - project.revisions.create - end + raise ArgumentError, 'Dev token required for dev projects' end end - private - - def add_resources_to_project(project, resource_map) - resource_map.each do |resource_type, resources| - project_log(project, "Inserting local project #{resource_type}", true) - resources.each do |resource| - resource = resource.with_indifferent_access - resource.delete(:project_id) - id = resource.delete(:id) - id_key = "#{resource_type}_id" - resource = ResourceAnonymizer.new(resource_type, resource).process - resource = ResourceTranslator.new(resource_type, resource, project: project).process - resource = ResourceCategorizer.new(resource_type, resource, project: project).process - project.send(resource_type).create(id_key => id).update(resource) - print '.' - end + def execute!(limit: nil) + timestamp = Time.now.utc.to_s.tr(':', '-') + Project.find_each do |project| + ProjectPopulator.new(project: project, com_client: com_client, dev_client: dev_client, timestamp: timestamp).execute!(limit: limit) + GC.start + puts GC.stat puts - project_log(project, "Inserted local project #{resource_type}", true) end end - def project_log(project, message, ongoing = false) - log = "[[#{Time.now.to_s}]]".tap do |msg| - msg << ' ' << message - msg << " for #{project.path}" - msg << '...' if ongoing - end - puts log - end - - def retrieve_resources(resource, project, limit) - client = client_for_project(project) - client.retrieve_resources(resource, project.fetch_path, limit) - end + private - def client_for_project(project) - project.dev? ? dev_client : client - end + attr_reader :com_client, :dev_client def dev_token_necessary? - Project.where(dev: true).any? + Project.exists?(dev: true) end end end diff --git a/lib/gitlab_insights/dev_usernames.rb b/lib/gitlab_insights/dev_usernames.rb index 931211b3151a7a023e86892e76013c660f620327..ab162ce3d96c567854404046bb2c3159b3727cc8 100644 --- a/lib/gitlab_insights/dev_usernames.rb +++ b/lib/gitlab_insights/dev_usernames.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module GitlabInsights module DevUsernames DEV_USERNAME_MAPPING = { @@ -25,6 +27,6 @@ module GitlabInsights 'gabriel' => 'brodock', 'jvargas' => 'jivanvl', 'phughes' => 'iamphill' - } + }.freeze end end diff --git a/lib/gitlab_insights/limit_period.rb b/lib/gitlab_insights/limit_period.rb index bba78a35aed5ede168f73af309b361af82d7c31d..570d3aef0897a1152e7250d7240b4f249eaa8507 100644 --- a/lib/gitlab_insights/limit_period.rb +++ b/lib/gitlab_insights/limit_period.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module GitlabInsights module LimitPeriod def limit_results(results, limit) diff --git a/lib/gitlab_insights/project_logger.rb b/lib/gitlab_insights/project_logger.rb new file mode 100644 index 0000000000000000000000000000000000000000..ac25e865dba57969788fccd59e4b5a339bb7afcb --- /dev/null +++ b/lib/gitlab_insights/project_logger.rb @@ -0,0 +1,24 @@ +# frozen_string_literal: true + +module GitlabInsights + class ProjectLogger + def self.log(project, message, ongoing = false) + self.new(project).log(message, ongoing) + end + + def initialize(project) + @project_path = project.respond_to?(:path) ? project.full_path : project.to_s + end + + def log(message, ongoing = false) + msg = +"[[#{Time.now.to_s}]] #{message} for #{project_path}" + msg << '...' if ongoing + + puts msg + end + + private + + attr_reader :project_path + end +end diff --git a/lib/gitlab_insights/project_populator.rb b/lib/gitlab_insights/project_populator.rb new file mode 100644 index 0000000000000000000000000000000000000000..eb9c10bad2a66703e971538854bb4b3428416594 --- /dev/null +++ b/lib/gitlab_insights/project_populator.rb @@ -0,0 +1,97 @@ +# frozen_string_literal: true + +require 'get_process_mem' +require 'memory_profiler' +require 'fileutils' + +module GitlabInsights + class ProjectPopulator + PROCESSORS_PIPELINES = [ + ResourceAnonymizer, + ResourceTranslator, + ResourceCategorizer + ].freeze + + def initialize(project:, com_client:, dev_client: nil, timestamp: nil) + @project = project + @api_client ||= project.dev? ? dev_client : com_client + @logger = ProjectLogger.new(project) + @timestamp = timestamp + end + + def execute!(limit: nil) + logger.log('=> Retrieving remote project resources', true) + + fetch_and_insert_resources(:issues, limit) + puts + fetch_and_insert_resources(:merge_requests, limit) + + project.revisions.create + end + + private + + attr_reader :project, :api_client, :logger + + def print_usage(description, before_mb = nil) + return if Rails.env.test? + + mb = GetProcessMem.new.mb + msg = "#{ description } - MEMORY USAGE(MB): #{ mb.round }" + + if before_mb + diff_mb = (mb - before_mb).round + msg += diff_mb.positive? ? " (+ #{diff_mb} MB)" : " (- #{diff_mb} MB)" + end + puts msg + + mb + end + + def print_usage_before_and_after(project_path, resource_type) + before_mb = print_usage("Before #{project_path} / #{resource_type}") + yield.tap do |result| + print_usage("After #{project_path} / #{resource_type}", before_mb) + end + end + + def fetch_and_insert_resources(resource_type, limit) + logger.log("==> Clearing local project #{resource_type}", true) + project.public_send(resource_type).clear + logger.log("==> Cleared local project #{resource_type}") + + logger.log("==> Inserting #{limit || 'all'} #{resource_type}", true) + inserted_ids = [] + report = nil + project_path = project.fetch_path + print_usage_before_and_after(project_path, resource_type) do + report = MemoryProfiler.report do + project_resources = Api::ProjectResources.new(project_path: project_path, resource_type: resource_type, api_client: api_client) + inserted_ids = project_resources.retrieve_resources(limit: limit) do |resource| + insert_resource(resource_type, resource) + end + end + end + + if Rails.env.development? || ENV['CI'] + FileUtils.mkdir_p("tmp/profiles/#{@timestamp}") + report.pretty_print(to_file: "tmp/profiles/#{@timestamp}/#{project_path.tr('/', '_')}-profile.txt") if report + end + logger.log("==> Inserted #{inserted_ids.size} #{resource_type}") + end + + def insert_resource(resource_type, resource) + resource[:"#{resource_type}_id"] = resource.delete('id') + resource = processors_pipeline(resource_type, resource) + project.send(resource_type).create(resource) + end + + def processors_pipeline(resource_type, resource) + PROCESSORS_PIPELINES.each do |processor| + resource = processor.new(resource_type, resource, project: project).process + end + + resource + end + end +end diff --git a/lib/gitlab_insights/regex.rb b/lib/gitlab_insights/regex.rb index 2691c93ba4beb7a4063011bf11885d465ac5f05d..2c6451ca71ae19e7015de28e79998d3f06317f28 100644 --- a/lib/gitlab_insights/regex.rb +++ b/lib/gitlab_insights/regex.rb @@ -1,5 +1,7 @@ +# frozen_string_literal: true + module GitlabInsights module Regex - MILESTONE_MATCHER = /\A\d+(\.\d+(\.\d+)?)?\z/ + MILESTONE_MATCHER = /\A\d+(\.\d+(\.\d+)?)?\z/.freeze end end diff --git a/lib/gitlab_insights/resource_anonymizer.rb b/lib/gitlab_insights/resource_anonymizer.rb index f2e1e6df71a0b3dc703ce32b138aa1bd03e6683f..fd28b5531c7613b2610b5ad72fea0680f60cb8c7 100644 --- a/lib/gitlab_insights/resource_anonymizer.rb +++ b/lib/gitlab_insights/resource_anonymizer.rb @@ -1,42 +1,42 @@ +# frozen_string_literal: true + module GitlabInsights class ResourceAnonymizer < ResourceModifier - WHITELISTED_ATTRIBUTES = [ - :issues_id, - :due_date, - :discussion_locked, - :time_stats, - :weight, - :merge_requests_id, - :iid, - :state, - :created_at, - :updated_at, - :closed_at, - :merged_at, - :upvotes, - :downvotes, - :labels, - :work_in_progress, - :milestone, - :merge_when_pipeline_succeeds, - :merge_status, - :user_notes_count, - :discussion_locked, - :should_remove_source_branch, - :force_remove_source_branch, - :time_stats, - :approvals_before_merge, - :squash, - :allow_maintainer_to_push, - :author + WHITELISTED_ATTRIBUTES = %w[ + issues_id + due_date + discussion_locked + time_stats + weight + merge_requests_id + iid + state + created_at + updated_at + closed_at + merged_at + upvotes + downvotes + labels + work_in_progress + milestone + merge_when_pipeline_succeeds + merge_status + user_notes_count + discussion_locked + should_remove_source_branch + force_remove_source_branch + time_stats + approvals_before_merge + squash + allow_maintainer_to_push + author ].freeze private def do_process - resource.each_key do |field| - resource.delete(field) unless WHITELISTED_ATTRIBUTES.include?(field.to_sym) - end + resource.slice(*WHITELISTED_ATTRIBUTES) end end end diff --git a/lib/gitlab_insights/resource_categorizer.rb b/lib/gitlab_insights/resource_categorizer.rb index c6044c8ec830dcb01c8781693368b12f298141f9..edca166ec9ab7c438270209b6f56c2bf04732a00 100644 --- a/lib/gitlab_insights/resource_categorizer.rb +++ b/lib/gitlab_insights/resource_categorizer.rb @@ -1,20 +1,21 @@ +# frozen_string_literal: true + module GitlabInsights class ResourceCategorizer < ResourceModifier - def initialize(resource_type, resource, project:) - super - end - private + def applicable? + project.dev? && resource_type == :merge_requests + end + def do_process add_security_label end def add_security_label - return unless project.dev? - return unless resource_type == :merge_requests - - resource[:labels] << 'security' unless resource[:labels].include?('security') + resource.tap do |r| + r['labels'] << 'security' unless r['labels'].include?('security') + end end end end diff --git a/lib/gitlab_insights/resource_modifier.rb b/lib/gitlab_insights/resource_modifier.rb index 7dc39fbfccc5581a41a7d8910c12d56e83f472dd..c53317fd27dde5f57496ebc2807327303f6db158 100644 --- a/lib/gitlab_insights/resource_modifier.rb +++ b/lib/gitlab_insights/resource_modifier.rb @@ -1,8 +1,10 @@ +# frozen_string_literal: true + module GitlabInsights class ResourceModifier - APPLICABLE_TYPES = [ - :issues, - :merge_requests + DEFAULT_APPLICABLE_TYPES = %i[ + issues + merge_requests ].freeze def initialize(resource_type, resource, project: nil) @@ -12,11 +14,11 @@ module GitlabInsights end def process - return unless applicable? - - do_process - - resource + if applicable? + do_process + else + resource + end end private @@ -24,7 +26,7 @@ module GitlabInsights attr_reader :resource_type, :resource, :project def applicable? - APPLICABLE_TYPES.include?(resource_type) + DEFAULT_APPLICABLE_TYPES.include?(resource_type) end def do_process diff --git a/lib/gitlab_insights/resource_translator.rb b/lib/gitlab_insights/resource_translator.rb index e0e4748f3fcc23969b2e2cd65e536951f49a30a5..edd9a41f5fa34937f33dd7c681c1597d68aba1cb 100644 --- a/lib/gitlab_insights/resource_translator.rb +++ b/lib/gitlab_insights/resource_translator.rb @@ -1,20 +1,21 @@ +# frozen_string_literal: true + module GitlabInsights class ResourceTranslator < ResourceModifier - def initialize(resource_type, resource, project:) - super - end - private + def applicable? + project.dev? && DevUsernames::DEV_USERNAME_MAPPING.key?(resource['author']['username']) + end + def do_process translate_username end def translate_username - return unless project.dev? - return unless DevUsernames::DEV_USERNAME_MAPPING.key?(resource[:author][:username]) - - resource[:author][:username] = DevUsernames::DEV_USERNAME_MAPPING[resource[:author][:username]] + resource.tap do |r| + r['author']['username'] = DevUsernames::DEV_USERNAME_MAPPING[r['author']['username']] + end end end end diff --git a/lib/tasks/gitlab_insights/populate.rake b/lib/tasks/gitlab_insights/populate.rake index 1c377e32f823e8f6ff008229ad3e43dadb3ba41b..207b493a9e7533d1cb17af7675e31bceace16310 100644 --- a/lib/tasks/gitlab_insights/populate.rake +++ b/lib/tasks/gitlab_insights/populate.rake @@ -1,29 +1,29 @@ namespace :gitlab_insights do DEFAULT_API_ENDPOINT = 'https://gitlab.com/api/v4' DEV_API_ENDPOINT = 'https://dev.gitlab.org/api/v4' - LIMIT = 500 + DEFAULT_LIMIT = 500 desc 'Clear resources and populate the DB with data for all projects' task :populate, [:token, :dev_token] => :environment do |_, args| args.with_defaults(dev_token: nil) - GitlabInsights::DbPopulator.new(client(args[:token]), dev_client(args[:dev_token])).execute! + GitlabInsights::DbPopulator.new(client(args[:token]), dev_client(args[:dev_token])).execute!(limit: ENV['LIMIT']) end desc 'Clear resources and populate the DB with data for all projects' task :populate_limited, [:token, :dev_token] => :environment do |_, args| args.with_defaults(dev_token: nil) - GitlabInsights::DbPopulator.new(client(args[:token]), dev_client(args[:dev_token])).execute!(LIMIT) + GitlabInsights::DbPopulator.new(client(args[:token]), dev_client(args[:dev_token])).execute!(limit: ENV['LIMIT'] || DEFAULT_LIMIT) end def client(token) - GitlabInsights::Api::Client.new(DEFAULT_API_ENDPOINT, token) + GitlabInsights::Api::Client.new(endpoint: DEFAULT_API_ENDPOINT, token: token) end def dev_client(token) if token - GitlabInsights::Api::Client.new(DEV_API_ENDPOINT, token) + GitlabInsights::Api::Client.new(endpoint: DEV_API_ENDPOINT, token: token) end end end diff --git a/spec/lib/gitlab_insights/api/project_resources_spec.rb b/spec/lib/gitlab_insights/api/project_resources_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..eea7626efe03590ea7fd86a138268e1a8bd8efea --- /dev/null +++ b/spec/lib/gitlab_insights/api/project_resources_spec.rb @@ -0,0 +1,104 @@ +require 'rails_helper' +require 'ostruct' + +RSpec.describe GitlabInsights::Api::ProjectResources do + let(:logger) { double(log: '') } + let(:resource_type) { :issues } + let!(:project) { create(:project) } + + subject { described_class.new(project_path: project.full_path, resource_type: resource_type, api_client: nil) } + + before do + allow(class_double('ProjectLogger').as_stubbed_const).to receive(:new).and_return(logger) + end + + context '#retrieve_resources' do + shared_examples 'resources retrieval for resource type' do |resource_type| + let(:api_page_class) do + Class.new(Array) do + def initialize(*args, has_next_page:) + super(*args) + @has_next_page = has_next_page + end + + def has_next_page? + @has_next_page + end + end + end + let(:api_page) { api_page_class.new(ApiResponses.public_send(resource_type).map { |r| OpenStruct.new(r) }, has_next_page: false) } + let(:api_empty_page) { api_page_class.new([], has_next_page: false) } + + context 'with no limit given' do + it 'requests 5 pages by default' do + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 1)).and_return(api_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 2)).and_return(api_empty_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 3)).and_return(api_empty_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 4)).and_return(api_empty_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 5)).and_return(api_empty_page) + + expect do + subject.retrieve_resources { |resource| project.public_send(resource_type).create(resource.slice(:iid)) } + end.to change { project.public_send(resource_type).count }.by(ApiResponses.public_send(resource_type).size) + end + end + + context 'with a limit == 1 given' do + it 'requests only 1 page' do + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 1)).and_return(api_page) + + expect do + subject.retrieve_resources(limit: 1) { |resource| project.public_send(resource_type).create(resource.slice(:iid)) } + end.to change { project.public_send(resource_type).count }.by(1) + end + end + + context 'with a limit == 150 given' do + it 'requests only 1 page' do + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 1)).and_return(api_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 2)).and_return(api_empty_page) + + subject.retrieve_resources(limit: 150) {} + end + end + + context 'with a limit == 600 given' do + it 'requests only 1 page' do + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 1)).and_return(api_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 2)).and_return(api_empty_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 3)).and_return(api_empty_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 4)).and_return(api_empty_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 5)).and_return(api_empty_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 6)).and_return(api_empty_page) + + subject.retrieve_resources(limit: 600) {} + end + end + + context 'with limit == 100 and no page_offset given' do + it 'starts at page 1' do + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 1)).and_return(api_page) + + subject.retrieve_resources(limit: 100) {} + end + end + + context 'with a limit == 200 and page_offset == 1 given' do + it 'starts at page 1 and fetches 2 pages' do + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 2)).and_return(api_empty_page) + expect(subject).to receive(:sync_api_call).with(described_class::DEFAULT_OPTIONS.merge(page: 3)).and_return(api_empty_page) + + subject.retrieve_resources(page_offset: 1, limit: 200) {} + end + end + end + + context 'for issues' do + it_behaves_like 'resources retrieval for resource type', :issues + end + + context 'for merge requests' do + it_behaves_like 'resources retrieval for resource type', :merge_requests + end + end +end diff --git a/spec/lib/gitlab_insights/db_populator_spec.rb b/spec/lib/gitlab_insights/db_populator_spec.rb index e067ae47b3db05d9e1afd8274f4f8e367362edac..6e5ffbccce665e174f83e68374b041ef312a55ee 100644 --- a/spec/lib/gitlab_insights/db_populator_spec.rb +++ b/spec/lib/gitlab_insights/db_populator_spec.rb @@ -1,23 +1,15 @@ require 'rails_helper' -require 'gitlab_insights/db_populator' RSpec.describe GitlabInsights::DbPopulator do - let(:client) { GitlabInsights::Api::Client.new('') } + let(:com_client) { GitlabInsights::Api::Client.new(endpoint: 'com') } let(:dev_client) { nil } let(:group) { create(:group) } let!(:project) { create(:project, group: group) } - let!(:issue) { create(:issue, issues_id: 999, project: project) } - let!(:original_issue_count) { project.issues.count } - subject { GitlabInsights::DbPopulator.new(client, dev_client) } + subject { described_class.new(com_client, dev_client) } - before do - allow(client).to receive(:retrieve_resources).with(:issues, project.full_path, nil).and_return(ApiResponses.issues) - allow(client).to receive(:retrieve_resources).with(:merge_requests, project.full_path, nil).and_return(ApiResponses.merge_requests) - end - - context '#new' do + describe '#new' do context 'with dev projects' do let!(:dev_project) { create(:project, :dev, group: group) } @@ -29,54 +21,14 @@ RSpec.describe GitlabInsights::DbPopulator do end end - context '#execute' do - context 'with dev projects' do - let!(:dev_project) { create(:project, :dev, group: group) } - let(:dev_client) { GitlabInsights::Api::Client.new('') } - - before do - allow(dev_client).to receive(:retrieve_resources).with(:issues, dev_project.override_path, nil).and_return(ApiResponses.issues) - allow(dev_client).to receive(:retrieve_resources).with(:merge_requests, dev_project.override_path, nil).and_return(ApiResponses.merge_requests) - end - - it 'calls the api for resources' do - expect(dev_client).to receive(:retrieve_resources).with(:issues, dev_project.override_path, nil) - expect(dev_client).to receive(:retrieve_resources).with(:merge_requests, dev_project.override_path, nil) - - subject.execute! - end - end - - it 'calls the api for resources' do - expect(client).to receive(:retrieve_resources).with(:issues, project.full_path, nil) - expect(client).to receive(:retrieve_resources).with(:merge_requests, project.full_path, nil) - - subject.execute! - end - - it 'adds resources to the project' do - subject.execute! - - expect(project.reload.issues.length).to eq(ApiResponses.issues.length) - expect(project.reload.merge_requests.length).to eq(ApiResponses.merge_requests.length) - end + describe '#execute!' do + it 'delegates to ProjectPopulator' do + project_populator = double - it 'creates a new revision' do - original_count = project.revisions.count + expect(GitlabInsights::ProjectPopulator).to receive(:new).with(project: project, com_client: com_client, dev_client: dev_client, timestamp: a_kind_of(String)).and_return(project_populator) + expect(project_populator).to receive(:execute!).with(limit: nil) subject.execute! - - expect(project.reload.revisions.count).to eq(original_count + 1) - end - - it 'removes existing resources' do - subject.execute! - - expect(project.reload.issues).not_to include(issue) - - expect do - issue.reload - end.to raise_error(ActiveRecord::RecordNotFound) end end end diff --git a/spec/lib/gitlab_insights/project_populator_spec.rb b/spec/lib/gitlab_insights/project_populator_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..de2d90d8ecb5fcf6d2507b7dd2074fd475b40e3b --- /dev/null +++ b/spec/lib/gitlab_insights/project_populator_spec.rb @@ -0,0 +1,80 @@ +require 'rails_helper' + +RSpec.describe GitlabInsights::ProjectPopulator do + let(:com_client) { GitlabInsights::Api::Client.new(endpoint: 'com') } + let(:dev_client) { nil } + + let(:group) { create(:group) } + let(:project) { create(:project, group: group) } + let(:dev_project) { create(:project, :dev, group: group) } + let(:api_issues_resources) { double } + let(:api_mrs_resources) { double } + let(:expected_limit) { nil } + let(:logger) { double(log: '') } + + subject { described_class.new(project: project, com_client: com_client, dev_client: dev_client) } + + before do + allow(class_double('ProjectLogger').as_stubbed_const).to receive(:new).and_return(logger) + allow(GitlabInsights::Api::ProjectResources).to receive(:new).with(project_path: project.full_path, resource_type: :issues, api_client: com_client).and_return(api_issues_resources) + allow(GitlabInsights::Api::ProjectResources).to receive(:new).with(project_path: project.full_path, resource_type: :merge_requests, api_client: com_client).and_return(api_mrs_resources) + allow(api_issues_resources).to receive(:retrieve_resources).with(limit: expected_limit).and_yield(ApiResponses.issues[0]).and_yield(ApiResponses.issues[1]).and_return(ApiResponses.issues.map { |issue| issue['id'] }) + allow(api_mrs_resources).to receive(:retrieve_resources).with(limit: expected_limit).and_yield(ApiResponses.merge_requests[0]).and_yield(ApiResponses.merge_requests[1]).and_return(ApiResponses.merge_requests.map { |issue| issue['id'] }) + end + + describe '#execute!' do + it 'calls the API for resources' do + expect(api_issues_resources).to receive(:retrieve_resources).with(limit: expected_limit).and_yield(ApiResponses.issues[0]).and_yield(ApiResponses.issues[1]).and_return(ApiResponses.issues.map { |issue| issue['id'] }) + expect(api_mrs_resources).to receive(:retrieve_resources).with(limit: expected_limit).and_yield(ApiResponses.merge_requests[0]).and_yield(ApiResponses.merge_requests[1]).and_return(ApiResponses.merge_requests.map { |issue| issue['id'] }) + + subject.execute! + end + + context 'with dev projects' do + let(:dev_client) { GitlabInsights::Api::Client.new(endpoint: 'dev') } + + subject { described_class.new(project: dev_project, com_client: com_client, dev_client: dev_client) } + + it 'instantiates GitlabInsights::Api::ProjectResources with the dev client' do + expect(GitlabInsights::Api::ProjectResources).to receive(:new).with(project_path: dev_project.override_path, resource_type: :issues, api_client: dev_client).and_return(api_issues_resources) + expect(GitlabInsights::Api::ProjectResources).to receive(:new).with(project_path: dev_project.override_path, resource_type: :merge_requests, api_client: dev_client).and_return(api_mrs_resources) + + subject.execute! + end + end + + it 'adds resources to the project' do + subject.execute! + + expect(project.reload.issues.size).to eq(ApiResponses.issues.size) + expect(project.reload.merge_requests.size).to eq(ApiResponses.merge_requests.size) + end + + it 'creates a new revision' do + expect { subject.execute! }.to change { project.revisions.count }.by(1) + end + + context 'with existing resources' do + let!(:issue) { create(:issue, issues_id: 666, project: project) } + let!(:merge_request) { create(:merge_request, merge_requests_id: 999, project: project) } + + it 'removes existing resources' do + subject.execute! + + expect(project.reload.issues).not_to include(issue) + + expect do + issue.reload + end.to raise_error(ActiveRecord::RecordNotFound) + end + end + + context 'with a given limit' do + let(:expected_limit) { 1000 } + + it 'requests the API with the given limit' do + subject.execute!(limit: 1000) + end + end + end +end diff --git a/spec/lib/gitlab_insights/resource_anonymizer_spec.rb b/spec/lib/gitlab_insights/resource_anonymizer_spec.rb index a2ae354f32c833f67027870878798fe5c465dbcb..4abdacac7881eaddb8c4eae10bbcd7eefe53ea06 100644 --- a/spec/lib/gitlab_insights/resource_anonymizer_spec.rb +++ b/spec/lib/gitlab_insights/resource_anonymizer_spec.rb @@ -1,14 +1,14 @@ require 'rails_helper' RSpec.describe GitlabInsights::ResourceAnonymizer do - [:issues, :merge_requests].each do |resource_type| - let(:resource) { ApiResponses.send(resource_type).first.with_indifferent_access } - let(:resource_keys) { resource.keys.map(&:to_sym) } + %i[issues merge_requests].each do |resource_type| + let(:resource) { ApiResponses.send(resource_type).first } + let(:resource_keys) { resource.keys } let(:anon_fields) { resource_keys - described_class::WHITELISTED_ATTRIBUTES } subject { described_class.new(resource_type, resource).process } - it 'deletes necessary fields' do + it 'deletes unnecessary fields' do anon_fields.each do |attr| expect(subject[attr]).to be_nil end diff --git a/spec/lib/gitlab_insights/resource_categorizer_spec.rb b/spec/lib/gitlab_insights/resource_categorizer_spec.rb index 5c5e40c3b82c7326ef14f61065712952ed4b755f..cf27cf1cc97836180f2bfa972fa727cf63e781ec 100644 --- a/spec/lib/gitlab_insights/resource_categorizer_spec.rb +++ b/spec/lib/gitlab_insights/resource_categorizer_spec.rb @@ -5,7 +5,7 @@ RSpec.describe GitlabInsights::ResourceCategorizer do let(:project) { create(:project, group: group) } context 'for issues' do - let(:resource) { ApiResponses.issues.first.with_indifferent_access } + let(:resource) { ApiResponses.issues.first } subject { described_class.new(:issues, resource, project: project).process } it 'does nothing' do @@ -27,9 +27,9 @@ RSpec.describe GitlabInsights::ResourceCategorizer do let(:project) { create(:project, :dev, group: group) } it 'adds the security label' do - expect(resource[:labels]).not_to include('security') + expect(resource['labels']).not_to include('security') - expect(subject[:labels]).to include('security') + expect(subject['labels']).to include('security') end end end diff --git a/spec/lib/gitlab_insights/resource_translator_spec.rb b/spec/lib/gitlab_insights/resource_translator_spec.rb index c0e956da14a15d65d38e91b0334c319ebc0e1a9e..f204c14db297c3ee12eabcfbb092ae086e5d7c54 100644 --- a/spec/lib/gitlab_insights/resource_translator_spec.rb +++ b/spec/lib/gitlab_insights/resource_translator_spec.rb @@ -4,8 +4,7 @@ RSpec.describe GitlabInsights::ResourceTranslator do [:issues, :merge_requests].each do |resource_type| let(:group) { create(:group) } let(:project) { create(:project, group: group) } - let(:resource) { ApiResponses.send(resource_type).first.with_indifferent_access } - let(:resource_keys) { resource.keys.map(&:to_sym) } + let(:resource) { ApiResponses.send(resource_type).first } subject { described_class.new(resource_type, resource, project: project).process } @@ -16,18 +15,18 @@ RSpec.describe GitlabInsights::ResourceTranslator do original_username = 'mark' expected_username = 'markglenfletcher' - resource[:author][:username] = original_username + resource['author']['username'] = original_username - expect(subject[:author][:username]).to eq(expected_username) + expect(subject['author']['username']).to eq(expected_username) end end it 'does not translate the username' do original_username = 'mark' - resource[:author][:username] = original_username + resource['author']['username'] = original_username - expect(subject[:author][:username]).to eq(original_username) + expect(subject['author']['username']).to eq(original_username) end end end diff --git a/spec/support/api_responses.rb b/spec/support/api_responses.rb index f9a2f954b3984f66a5e61e49c83e346cdc203146..b784728c4006af83c998200a6cab26a931440597 100644 --- a/spec/support/api_responses.rb +++ b/spec/support/api_responses.rb @@ -1,23 +1,58 @@ class ApiResponses class << self def issues - [{"id"=>9697780, "iid"=>4, "project_id"=>1143462, "title"=>"Issue with attachment", "description"=>"![logo](/uploads/697aba05d14849dbad80e004fa9da999/logo.png)", "state"=>"opened", "created_at"=>"2018-03-13T10:57:21.004Z", "updated_at"=>"2018-03-13T10:57:31.023Z", "closed_at"=>nil, "labels"=>[], "milestone"=>nil, "assignees"=>[], "author"=>{"id"=>419655, "name"=>"Mark Fletcher", "username"=>"markglenfletcher", "state"=>"active", "avatar_url"=>"https://assets.gitlab-static.net/uploads/-/system/user/avatar/419655/avatar.png", "web_url"=>"https://gitlab.com/markglenfletcher"}, "assignee"=>nil, "user_notes_count"=>0, "upvotes"=>0, "downvotes"=>0, "due_date"=>nil, "confidential"=>false, "discussion_locked"=>nil, "web_url"=>"https://gitlab.com/markglenfletcher/test-ci-project/issues/4", "time_stats"=>{"time_estimate"=>0, "total_time_spent"=>0, "human_time_estimate"=>nil, "human_total_time_spent"=>nil}, "weight"=>nil}, {"id"=>2742087, "iid"=>3, "project_id"=>1143462, "title"=>"012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234", "description"=>"", "state"=>"opened", "created_at"=>"2016-08-11T10:44:47.519Z", "updated_at"=>"2016-12-22T16:05:53.984Z", "closed_at"=>nil, "labels"=>[], "milestone"=>nil, "assignees"=>[], "author"=>{"id"=>419655, "name"=>"Mark Fletcher", "username"=>"markglenfletcher", "state"=>"active", "avatar_url"=>"https://assets.gitlab-static.net/uploads/-/system/user/avatar/419655/avatar.png", "web_url"=>"https://gitlab.com/markglenfletcher"}, "assignee"=>nil, "user_notes_count"=>2, "upvotes"=>0, "downvotes"=>0, "due_date"=>nil, "confidential"=>false, "discussion_locked"=>nil, "web_url"=>"https://gitlab.com/markglenfletcher/test-ci-project/issues/3", "time_stats"=>{"time_estimate"=>0, "total_time_spent"=>0, "human_time_estimate"=>nil, "human_total_time_spent"=>nil}, "weight"=>nil}] + [ + { + "id"=>9697780, "iid"=>4, "project_id"=>1143462, "title"=>"Issue with attachment", "description"=>"![logo](/uploads/697aba05d14849dbad80e004fa9da999/logo.png)", "state"=>"opened", "created_at"=>"2018-03-13T10:57:21.004Z", "updated_at"=>"2018-03-13T10:57:31.023Z", "closed_at"=>nil, "labels"=>[], "milestone"=>nil, "assignees"=>[], "author"=>{"id"=>419655, "name"=>"Mark Fletcher", "username"=>"markglenfletcher", "state"=>"active", "avatar_url"=>"https://assets.gitlab-static.net/uploads/-/system/user/avatar/419655/avatar.png", "web_url"=>"https://gitlab.com/markglenfletcher"}, "assignee"=>nil, "user_notes_count"=>0, "upvotes"=>0, "downvotes"=>0, "due_date"=>nil, "confidential"=>false, "discussion_locked"=>nil, "web_url"=>"https://gitlab.com/markglenfletcher/test-ci-project/issues/4", "time_stats"=>{"time_estimate"=>0, "total_time_spent"=>0, "human_time_estimate"=>nil, "human_total_time_spent"=>nil}, "weight"=>nil + }, + { + "id"=>2742087, "iid"=>3, "project_id"=>1143462, "title"=>"012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234", "description"=>"", "state"=>"opened", "created_at"=>"2016-08-11T10:44:47.519Z", "updated_at"=>"2016-12-22T16:05:53.984Z", "closed_at"=>nil, "labels"=>[], "milestone"=>nil, "assignees"=>[], "author"=>{"id"=>419655, "name"=>"Mark Fletcher", "username"=>"markglenfletcher", "state"=>"active", "avatar_url"=>"https://assets.gitlab-static.net/uploads/-/system/user/avatar/419655/avatar.png", "web_url"=>"https://gitlab.com/markglenfletcher"}, "assignee"=>nil, "user_notes_count"=>2, "upvotes"=>0, "downvotes"=>0, "due_date"=>nil, "confidential"=>false, "discussion_locked"=>nil, "web_url"=>"https://gitlab.com/markglenfletcher/test-ci-project/issues/3", "time_stats"=>{"time_estimate"=>0, "total_time_spent"=>0, "human_time_estimate"=>nil, "human_total_time_spent"=>nil}, "weight"=>nil + } + ] end def labels - [{"id"=>514708, "name"=>"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", "color"=>"#428bca", "description"=>"", "open_issues_count"=>0, "closed_issues_count"=>0, "open_merge_requests_count"=>0, "priority"=>nil, "subscribed"=>false}, {"id"=>479411, "name"=>"Won@t fix", "color"=>"#428bca", "description"=>"Won't fix", "open_issues_count"=>1, "closed_issues_count"=>0, "open_merge_requests_count"=>0, "priority"=>nil, "subscribed"=>false}] + [ + { + "id"=>514708, "name"=>"01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890", "color"=>"#428bca", "description"=>"", "open_issues_count"=>0, "closed_issues_count"=>0, "open_merge_requests_count"=>0, "priority"=>nil, "subscribed"=>false + }, + { + "id"=>479411, "name"=>"Won@t fix", "color"=>"#428bca", "description"=>"Won't fix", "open_issues_count"=>1, "closed_issues_count"=>0, "open_merge_requests_count"=>0, "priority"=>nil, "subscribed"=>false + } + ] end def merge_requests - [{"id"=>8998122, "iid"=>18281, "project_id"=>13083, "title"=>"[Rails5] Fix running spinach tests", "description"=>"## What does this MR do?\n\n1. Adds support for `RAILS5=1|true` for the `bin/spinach` command.\n2. Synchronizes used spinach versions both for rails4 and rails5.\n\nFor rails5 it was accidently used spinach 0.10.1 instead of 0.8.10.\nThat brought some problems on running spinach tests.\n\nExample of failure message:\n\n```\nNoMethodError: undefined method `line' for #\nDid you mean? lines\n /builds/gitlab-org/gitlab-foss/features/support/env.rb:52:in `before_scenario_run'\n```\n\nhttps://gitlab.com/gitlab-org/gitlab-foss/-/jobs/62129156 \n\n## Are there points in the code the reviewer needs to double check?\n\nNo. \n\n## Why was this MR needed?\n\nMigration to Rails 5.0.\n\n## Screenshots (if relevant)\n\nNo. \n\n## Does this MR meet the acceptance criteria?\n\n- [ ] [Changelog entry](https://docs.gitlab.com/ee/development/changelog.html) added, if necessary\n- [ ] [Documentation created/updated](https://docs.gitlab.com/ee/development/doc_styleguide.html)\n- [ ] API support added\n- [ ] Tests added for this feature/bug\n- Review\n - [ ] Has been reviewed by UX\n - [ ] Has been reviewed by Frontend\n - [ ] Has been reviewed by Backend\n - [ ] Has been reviewed by Database\n- [ ] Conform by the [merge request performance guides](https://docs.gitlab.com/ee/development/merge_request_performance_guidelines.html)\n- [ ] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab/blob/master/CONTRIBUTING.md#style-guides)\n- [ ] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)\n- [ ] Internationalization required/considered\n- [ ] End-to-end tests pass (`package-and-qa` manual pipeline job)\n\n## What are the relevant issue numbers?\n\n#14286 and !12841", "state"=>"opened", "created_at"=>"2018-04-10T11:25:53.532Z", "updated_at"=>"2018-04-10T11:31:21.192Z", "target_branch"=>"master", "source_branch"=>"blackst0ne-rails5-fix-spinach", "upvotes"=>0, "downvotes"=>0, "author"=>{"id"=>86853, "name"=>"blackst0ne", "username"=>"blackst0ne", "state"=>"active", "avatar_url"=>"https://secure.gravatar.com/avatar/1613b1d2412639606af3866da674f0e1?s=80&d=identicon", "web_url"=>"https://gitlab.com/blackst0ne"}, "assignee"=>{"id"=>128633, "name"=>"Rémy Coutable", "username"=>"rymai", "state"=>"active", "avatar_url"=>"https://secure.gravatar.com/avatar/263da227929cc0035cb0eba512bcf81a?s=80&d=identicon", "web_url"=>"https://gitlab.com/rymai"}, "source_project_id"=>13083, "target_project_id"=>13083, "labels"=>["Community contribution", "Edge", "backend", "dependency update", "rails5", "test"], "work_in_progress"=>false, "milestone"=>{"id"=>445863, "iid"=>12, "group_id"=>9970, "title"=>"10.8", "description"=>"", "state"=>"active", "created_at"=>"2018-01-16T17:05:23.570Z", "updated_at"=>"2018-01-16T17:05:23.570Z", "due_date"=>"2018-05-22", "start_date"=>"2018-04-08"}, "merge_when_pipeline_succeeds"=>true, "merge_status"=>"can_be_merged", "sha"=>"1a455f3d5c2607c81af4f45a971f310d9210c2ba", "merge_commit_sha"=>nil, "user_notes_count"=>2, "discussion_locked"=>nil, "should_remove_source_branch"=>true, "force_remove_source_branch"=>false, "web_url"=>"https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/18281", "time_stats"=>{"time_estimate"=>0, "total_time_spent"=>0, "human_time_estimate"=>nil, "human_total_time_spent"=>nil}, "approvals_before_merge"=>nil, "squash"=>true}, {"id"=>9001792, "iid"=>18282, "project_id"=>13083, "title"=>"Explain Auto DevOps better", "description"=>"## What does this MR do?\n\nPort of https://gitlab.com/gitlab-org/gitlab/merge_requests/5290", "state"=>"opened", "created_at"=>"2018-04-10T12:28:20.934Z", "updated_at"=>"2018-04-10T12:28:50.963Z", "target_branch"=>"master", "source_branch"=>"docs/explain-auto-devops", "upvotes"=>0, "downvotes"=>0, "author"=>{"id"=>3585, "name"=>"Achilleas Pipinellis", "username"=>"axil", "state"=>"active", "avatar_url"=>"https://assets.gitlab-static.net/uploads/-/system/user/avatar/3585/avatar.png", "web_url"=>"https://gitlab.com/axil"}, "assignee"=>{"id"=>236961, "name"=>"Marcia Ramos", "username"=>"marcia", "state"=>"active", "avatar_url"=>"https://assets.gitlab-static.net/uploads/-/system/user/avatar/236961/avatar.png", "web_url"=>"https://gitlab.com/marcia"}, "source_project_id"=>13083, "target_project_id"=>13083, "labels"=>["Documentation", "auto devops"], "work_in_progress"=>false, "milestone"=>{"id"=>445863, "iid"=>12, "group_id"=>9970, "title"=>"10.8", "description"=>"", "state"=>"active", "created_at"=>"2018-01-16T17:05:23.570Z", "updated_at"=>"2018-01-16T17:05:23.570Z", "due_date"=>"2018-05-22", "start_date"=>"2018-04-08"}, "merge_when_pipeline_succeeds"=>false, "merge_status"=>"can_be_merged", "sha"=>"f2c684960ce52e7ad05ccd836e10cb2f4ccdf74d", "merge_commit_sha"=>nil, "user_notes_count"=>0, "discussion_locked"=>nil, "should_remove_source_branch"=>nil, "force_remove_source_branch"=>true, "web_url"=>"https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/18282", "time_stats"=>{"time_estimate"=>0, "total_time_spent"=>0, "human_time_estimate"=>nil, "human_total_time_spent"=>nil}, "approvals_before_merge"=>nil, "squash"=>false}] + [ + { + "id"=>8998122, "iid"=>18281, "project_id"=>13083, "title"=>"[Rails5] Fix running spinach tests", "description"=>"## What does this MR do?\n\n1. Adds support for `RAILS5=1|true` for the `bin/spinach` command.\n2. Synchronizes used spinach versions both for rails4 and rails5.\n\nFor rails5 it was accidently used spinach 0.10.1 instead of 0.8.10.\nThat brought some problems on running spinach tests.\n\nExample of failure message:\n\n```\nNoMethodError: undefined method `line' for #\nDid you mean? lines\n /builds/gitlab-org/gitlab-foss/features/support/env.rb:52:in `before_scenario_run'\n```\n\nhttps://gitlab.com/gitlab-org/gitlab-foss/-/jobs/62129156 \n\n## Are there points in the code the reviewer needs to double check?\n\nNo. \n\n## Why was this MR needed?\n\nMigration to Rails 5.0.\n\n## Screenshots (if relevant)\n\nNo. \n\n## Does this MR meet the acceptance criteria?\n\n- [ ] [Changelog entry](https://docs.gitlab.com/ee/development/changelog.html) added, if necessary\n- [ ] [Documentation created/updated](https://docs.gitlab.com/ee/development/doc_styleguide.html)\n- [ ] API support added\n- [ ] Tests added for this feature/bug\n- Review\n - [ ] Has been reviewed by UX\n - [ ] Has been reviewed by Frontend\n - [ ] Has been reviewed by Backend\n - [ ] Has been reviewed by Database\n- [ ] Conform by the [merge request performance guides](https://docs.gitlab.com/ee/development/merge_request_performance_guidelines.html)\n- [ ] Conform by the [style guides](https://gitlab.com/gitlab-org/gitlab/blob/master/CONTRIBUTING.md#style-guides)\n- [ ] [Squashed related commits together](https://git-scm.com/book/en/Git-Tools-Rewriting-History#Squashing-Commits)\n- [ ] Internationalization required/considered\n- [ ] End-to-end tests pass (`package-and-qa` manual pipeline job)\n\n## What are the relevant issue numbers?\n\n#14286 and !12841", "state"=>"opened", "created_at"=>"2018-04-10T11:25:53.532Z", "updated_at"=>"2018-04-10T11:31:21.192Z", "target_branch"=>"master", "source_branch"=>"blackst0ne-rails5-fix-spinach", "upvotes"=>0, "downvotes"=>0, "author"=>{"id"=>86853, "name"=>"blackst0ne", "username"=>"blackst0ne", "state"=>"active", "avatar_url"=>"https://secure.gravatar.com/avatar/1613b1d2412639606af3866da674f0e1?s=80&d=identicon", "web_url"=>"https://gitlab.com/blackst0ne"}, "assignee"=>{"id"=>128633, "name"=>"Rémy Coutable", "username"=>"rymai", "state"=>"active", "avatar_url"=>"https://secure.gravatar.com/avatar/263da227929cc0035cb0eba512bcf81a?s=80&d=identicon", "web_url"=>"https://gitlab.com/rymai"}, "source_project_id"=>13083, "target_project_id"=>13083, "labels"=>["Community contribution", "Edge", "backend", "dependency update", "rails5", "test"], "work_in_progress"=>false, "milestone"=>{"id"=>445863, "iid"=>12, "group_id"=>9970, "title"=>"10.8", "description"=>"", "state"=>"active", "created_at"=>"2018-01-16T17:05:23.570Z", "updated_at"=>"2018-01-16T17:05:23.570Z", "due_date"=>"2018-05-22", "start_date"=>"2018-04-08"}, "merge_when_pipeline_succeeds"=>true, "merge_status"=>"can_be_merged", "sha"=>"1a455f3d5c2607c81af4f45a971f310d9210c2ba", "merge_commit_sha"=>nil, "user_notes_count"=>2, "discussion_locked"=>nil, "should_remove_source_branch"=>true, "force_remove_source_branch"=>false, "web_url"=>"https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/18281", "time_stats"=>{"time_estimate"=>0, "total_time_spent"=>0, "human_time_estimate"=>nil, "human_total_time_spent"=>nil}, "approvals_before_merge"=>nil, "squash"=>true + }, + { + "id"=>9001792, "iid"=>18282, "project_id"=>13083, "title"=>"Explain Auto DevOps better", "description"=>"## What does this MR do?\n\nPort of https://gitlab.com/gitlab-org/gitlab/merge_requests/5290", "state"=>"opened", "created_at"=>"2018-04-10T12:28:20.934Z", "updated_at"=>"2018-04-10T12:28:50.963Z", "target_branch"=>"master", "source_branch"=>"docs/explain-auto-devops", "upvotes"=>0, "downvotes"=>0, "author"=>{"id"=>3585, "name"=>"Achilleas Pipinellis", "username"=>"axil", "state"=>"active", "avatar_url"=>"https://assets.gitlab-static.net/uploads/-/system/user/avatar/3585/avatar.png", "web_url"=>"https://gitlab.com/axil"}, "assignee"=>{"id"=>236961, "name"=>"Marcia Ramos", "username"=>"marcia", "state"=>"active", "avatar_url"=>"https://assets.gitlab-static.net/uploads/-/system/user/avatar/236961/avatar.png", "web_url"=>"https://gitlab.com/marcia"}, "source_project_id"=>13083, "target_project_id"=>13083, "labels"=>["Documentation", "auto devops"], "work_in_progress"=>false, "milestone"=>{"id"=>445863, "iid"=>12, "group_id"=>9970, "title"=>"10.8", "description"=>"", "state"=>"active", "created_at"=>"2018-01-16T17:05:23.570Z", "updated_at"=>"2018-01-16T17:05:23.570Z", "due_date"=>"2018-05-22", "start_date"=>"2018-04-08"}, "merge_when_pipeline_succeeds"=>false, "merge_status"=>"can_be_merged", "sha"=>"f2c684960ce52e7ad05ccd836e10cb2f4ccdf74d", "merge_commit_sha"=>nil, "user_notes_count"=>0, "discussion_locked"=>nil, "should_remove_source_branch"=>nil, "force_remove_source_branch"=>true, "web_url"=>"https://gitlab.com/gitlab-org/gitlab-foss/merge_requests/18282", "time_stats"=>{"time_estimate"=>0, "total_time_spent"=>0, "human_time_estimate"=>nil, "human_total_time_spent"=>nil}, "approvals_before_merge"=>nil, "squash"=>false + } + ] end def milestones - [{"id"=>339386, "iid"=>44, "project_id"=>13083, "title"=>"10.0", "description"=>"", "state"=>"closed", "created_at"=>"2017-07-03T13:55:50.520Z", "updated_at"=>"2018-02-26T11:26:06.808Z", "due_date"=>"2017-09-22", "start_date"=>"2017-08-08"}, {"id"=>315839, "iid"=>41, "project_id"=>13083, "title"=>"9.5", "description"=>"", "state"=>"active", "created_at"=>"2017-05-17T16:56:47.216Z", "updated_at"=>"2017-05-17T16:56:47.216Z", "due_date"=>"2017-08-22", "start_date"=>"2017-07-08"}] + [ + { + "id"=>339386, "iid"=>44, "project_id"=>13083, "title"=>"10.0", "description"=>"", "state"=>"closed", "created_at"=>"2017-07-03T13:55:50.520Z", "updated_at"=>"2018-02-26T11:26:06.808Z", "due_date"=>"2017-09-22", "start_date"=>"2017-08-08" + }, + { + "id"=>315839, "iid"=>41, "project_id"=>13083, "title"=>"9.5", "description"=>"", "state"=>"active", "created_at"=>"2017-05-17T16:56:47.216Z", "updated_at"=>"2017-05-17T16:56:47.216Z", "due_date"=>"2017-08-22", "start_date"=>"2017-07-08" + } + ] end def pipelines - [{"id"=>20260210, "sha"=>"707e55ab575718921e2102d5d58a0e53790502f8", "ref"=>"fix/gb/fix-pipeline-statuses-illustrations", "status"=>"running"}, {"id"=>20259895, "sha"=>"daf408129f3feb5348b564c5912c157dcb051d57", "ref"=>"improve-jobs-queuing-time-metric", "status"=>"running"}] + [ + { + "id"=>20260210, "sha"=>"707e55ab575718921e2102d5d58a0e53790502f8", "ref"=>"fix/gb/fix-pipeline-statuses-illustrations", "status"=>"running" + }, + { + "id"=>20259895, "sha"=>"daf408129f3feb5348b564c5912c157dcb051d57", "ref"=>"improve-jobs-queuing-time-metric", "status"=>"running" + } + ] end end end