From 1e62a13968cc4351684f919630cd426e20fc022a Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Mon, 28 Nov 2016 16:55:31 +0100 Subject: [PATCH 1/2] Improve pipeline fixtures --- db/fixtures/development/14_pipelines.rb | 63 +++++++++++++++++-------- 1 file changed, 44 insertions(+), 19 deletions(-) diff --git a/db/fixtures/development/14_pipelines.rb b/db/fixtures/development/14_pipelines.rb index 08ad3097d343..a019660e5f2f 100644 --- a/db/fixtures/development/14_pipelines.rb +++ b/db/fixtures/development/14_pipelines.rb @@ -1,26 +1,50 @@ class Gitlab::Seeder::Pipelines STAGES = %w[build test deploy notify] BUILDS = [ - { name: 'build:linux', stage: 'build', status: :success }, - { name: 'build:osx', stage: 'build', status: :success }, - { name: 'rspec:linux 0 3', stage: 'test', status: :success }, - { name: 'rspec:linux 1 3', stage: 'test', status: :success }, - { name: 'rspec:linux 2 3', stage: 'test', status: :success }, - { name: 'rspec:windows 0 3', stage: 'test', status: :success }, - { name: 'rspec:windows 1 3', stage: 'test', status: :success }, - { name: 'rspec:windows 2 3', stage: 'test', status: :success }, - { name: 'rspec:windows 2 3', stage: 'test', status: :success }, - { name: 'rspec:osx', stage: 'test', status_event: :success }, - { name: 'spinach:linux', stage: 'test', status: :success }, - { name: 'spinach:osx', stage: 'test', status: :failed, allow_failure: true}, - { name: 'env:alpha', stage: 'deploy', environment: 'alpha', status: :pending }, - { name: 'env:beta', stage: 'deploy', environment: 'beta', status: :running }, - { name: 'env:gamma', stage: 'deploy', environment: 'gamma', status: :canceled }, - { name: 'staging', stage: 'deploy', environment: 'staging', status_event: :success, options: { environment: { on_stop: 'stop staging' } } }, - { name: 'stop staging', stage: 'deploy', environment: 'staging', when: 'manual', status: :skipped }, - { name: 'production', stage: 'deploy', environment: 'production', when: 'manual', status: :skipped }, + # build stage + { name: 'build:linux', stage: 'build', status: :success, + queued_at: 10.hour.ago, started_at: 9.hour.ago, finished_at: 8.hour.ago }, + { name: 'build:osx', stage: 'build', status: :success, + queued_at: 10.hour.ago, started_at: 10.hour.ago, finished_at: 9.hour.ago }, + + # test stage + { name: 'rspec:linux 0 3', stage: 'test', status: :success, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + { name: 'rspec:linux 1 3', stage: 'test', status: :success, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + { name: 'rspec:linux 2 3', stage: 'test', status: :success, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + { name: 'rspec:windows 0 3', stage: 'test', status: :success, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + { name: 'rspec:windows 1 3', stage: 'test', status: :success, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + { name: 'rspec:windows 2 3', stage: 'test', status: :success, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + { name: 'rspec:windows 2 3', stage: 'test', status: :success, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + { name: 'rspec:osx', stage: 'test', status_event: :success, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + { name: 'spinach:linux', stage: 'test', status: :success, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + { name: 'spinach:osx', stage: 'test', status: :failed, allow_failure: true, + queued_at: 8.hour.ago, started_at: 8.hour.ago, finished_at: 7.hour.ago }, + + # deploy stage + { name: 'staging', stage: 'deploy', environment: 'staging', status_event: :success, + options: { environment: { action: 'start', on_stop: 'stop staging' } }, + queued_at: 7.hour.ago, started_at: 6.hour.ago, finished_at: 4.hour.ago }, + { name: 'stop staging', stage: 'deploy', environment: 'staging', + when: 'manual', status: :skipped }, + { name: 'production', stage: 'deploy', environment: 'production', + when: 'manual', status: :skipped }, + + # notify stage { name: 'slack', stage: 'notify', when: 'manual', status: :created }, ] + EXTERNAL_JOBS = [ + { name: 'jenkins', stage: 'test', status: :success, + queued_at: 7.hour.ago, started_at: 6.hour.ago, finished_at: 4.hour.ago }, + ] def initialize(project) @project = project @@ -30,11 +54,12 @@ def seed! pipelines.each do |pipeline| begin BUILDS.each { |opts| build_create!(pipeline, opts) } - commit_status_create!(pipeline, name: 'jenkins', stage: 'test', status: :success) + EXTERNAL_JOBS.each { |opts| commit_status_create!(pipeline, opts) } print '.' rescue ActiveRecord::RecordInvalid print 'F' ensure + pipeline.update_duration pipeline.update_status end end -- GitLab From 54ccd409c200b13f7de4d349a5b4f05912e90167 Mon Sep 17 00:00:00 2001 From: Kamil Trzcinski Date: Tue, 29 Nov 2016 18:47:18 +0100 Subject: [PATCH 2/2] Rough implementation of build minutes for shared runners [ci skip] --- .../admin/application_settings_controller.rb | 1 + app/models/application_setting.rb | 3 ++ app/models/ci/build.rb | 6 +++ app/models/project.rb | 21 ++++++++- app/models/project_metrics.rb | 5 +++ app/services/ci/register_build_service.rb | 43 +++++++++++++------ app/services/update_build_minutes_service.rb | 10 +++++ .../application_settings/_form.html.haml | 7 +++ app/views/admin/projects/show.html.haml | 14 ++++++ app/views/projects/builds/show.html.haml | 13 ++++++ .../clear_shared_runner_minutes_worker.rb | 8 ++++ config/initializers/1_settings.rb | 4 ++ ...runners_minutes_to_application_settings.rb | 9 ++++ ...hared_runners_minutes_limit_to_projects.rb | 9 ++++ ...1129161957_create_table_project_metrics.rb | 18 ++++++++ ...1129162216_add_index_to_project_metrics.rb | 11 +++++ lib/api/entities.rb | 1 + 17 files changed, 170 insertions(+), 13 deletions(-) create mode 100644 app/models/project_metrics.rb create mode 100644 app/services/update_build_minutes_service.rb create mode 100644 app/workers/clear_shared_runner_minutes_worker.rb create mode 100644 db/migrate/20161129161815_add_shared_runners_minutes_to_application_settings.rb create mode 100644 db/migrate/20161129161913_add_shared_runners_minutes_limit_to_projects.rb create mode 100644 db/migrate/20161129161957_create_table_project_metrics.rb create mode 100644 db/migrate/20161129162216_add_index_to_project_metrics.rb diff --git a/app/controllers/admin/application_settings_controller.rb b/app/controllers/admin/application_settings_controller.rb index b81842e319b2..b31eb49185d0 100644 --- a/app/controllers/admin/application_settings_controller.rb +++ b/app/controllers/admin/application_settings_controller.rb @@ -93,6 +93,7 @@ def application_setting_params :user_oauth_applications, :user_default_external, :shared_runners_enabled, + :shared_runners_minutes, :shared_runners_text, :max_artifacts_size, :metrics_enabled, diff --git a/app/models/application_setting.rb b/app/models/application_setting.rb index bf463a3b6bbd..312280c3b32d 100644 --- a/app/models/application_setting.rb +++ b/app/models/application_setting.rb @@ -107,6 +107,9 @@ class ApplicationSetting < ActiveRecord::Base presence: true, numericality: { only_integer: true, greater_than: :housekeeping_full_repack_period } + validates :shared_runners_minutes, + numericality: { greater_than_or_equal_to: 0 } + validates_each :restricted_visibility_levels do |record, attr, value| unless value.nil? value.each do |level| diff --git a/app/models/ci/build.rb b/app/models/ci/build.rb index e7d33bd26db6..ef63501d1ed3 100644 --- a/app/models/ci/build.rb +++ b/app/models/ci/build.rb @@ -93,6 +93,12 @@ def retry(build, user = nil) end end + after_transition any => [:success, :failed, :canceled] do |build| + build.run_after_commit do + UpdateBuildMinutesService.new(project, nil).execute(build) + end + end + after_transition any => [:success] do |build| build.run_after_commit do BuildSuccessWorker.perform_async(id) diff --git a/app/models/project.rb b/app/models/project.rb index f8a54324341b..d4cd398c9345 100644 --- a/app/models/project.rb +++ b/app/models/project.rb @@ -28,6 +28,8 @@ class BoardLimitExceeded < StandardError; end :merge_requests_enabled?, :issues_enabled?, to: :project_feature, allow_nil: true + delegate :shared_runners_minutes, to: :project_metrics, allow_nil: true + default_value_for :archived, false default_value_for :visibility_level, gitlab_config_features.visibility_level default_value_for :container_registry_enabled, gitlab_config_features.container_registry @@ -146,6 +148,7 @@ def update_forks_visibility_level has_one :import_data, dependent: :destroy, class_name: "ProjectImportData" has_one :project_feature, dependent: :destroy + has_one :project_metrics, dependent: :destroy has_many :commit_statuses, dependent: :destroy, foreign_key: :gl_project_id has_many :pipelines, dependent: :destroy, class_name: 'Ci::Pipeline', foreign_key: :gl_project_id @@ -1165,7 +1168,9 @@ def any_runners?(&block) return true end - shared_runners_enabled? && Ci::Runner.shared.active.any?(&block) + shared_runners_enabled? && + !shared_runners_minutes_used? && + Ci::Runner.shared.active.any?(&block) end def valid_runners_token?(token) @@ -1346,6 +1351,20 @@ def environments_recently_updated_on_branch(branch) end end + def shared_runners_minutes_limit + read_attribute(:shared_runners_minutes_limit) || current_application_settings.shared_runners_minutes + end + + def shared_runners_minutes_limit_enabled? + shared_runners_minutes_limit.nonzero? + end + + def shared_runners_minutes_used? + shared_runners_enabled? && + shared_runners_minutes_limit_enabled? && + shared_runners_minutes.to_i < shared_runners_minutes_limit + end + private def pushes_since_gc_redis_key diff --git a/app/models/project_metrics.rb b/app/models/project_metrics.rb new file mode 100644 index 000000000000..35f3c713de2e --- /dev/null +++ b/app/models/project_metrics.rb @@ -0,0 +1,5 @@ +class ProjectMetrics < ActiveRecord::Base + belongs_to :project + + validates :project, presence: true +end diff --git a/app/services/ci/register_build_service.rb b/app/services/ci/register_build_service.rb index 74b5ebf372b1..a9a663f528a8 100644 --- a/app/services/ci/register_build_service.rb +++ b/app/services/ci/register_build_service.rb @@ -2,24 +2,16 @@ module Ci # This class responsible for assigning # proper pending build to runner on runner API request class RegisterBuildService + include CurrentSettings + def execute(current_runner) builds = Ci::Build.pending.unstarted builds = if current_runner.shared? - builds. - # don't run projects which have not enabled shared runners and builds - joins(:project).where(projects: { shared_runners_enabled: true }). - joins('LEFT JOIN project_features ON ci_builds.gl_project_id = project_features.project_id'). - - # this returns builds that are ordered by number of running builds - # we prefer projects that don't use shared runners at all - joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.gl_project_id=project_builds.gl_project_id"). - where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0'). - order('COALESCE(project_builds.running_builds, 0) ASC', 'ci_builds.id ASC') + builds_for_shared_runner_with_build_minutes else - # do run projects which are only assigned to this runner (FIFO) - builds.where(project: current_runner.projects.with_builds_enabled).order('created_at ASC') + builds_for_specific_runner end build = builds.find do |build| @@ -41,9 +33,36 @@ def execute(current_runner) private + def builds_for_shared_runner + new_builds. + # don't run projects which have not enabled shared runners and builds + joins(:project).where(projects: { shared_runners_enabled: true }). + joins('LEFT JOIN project_features ON ci_builds.gl_project_id = project_features.project_id'). + where('project_features.builds_access_level IS NULL or project_features.builds_access_level > 0'). + + # select projects with allowed number of shared runner minutes + joins('LEFT JOIN project_metrics ON ci_builds.gl_project_id = project_metrics.project_id'). + where('COALESCE(projects.shared_runner_minutes_limit, ?, 0) > 0 AND ' \ + 'COALESCE(project_metrics.shared_runner_minutes, 0) < COALESCE(projects.shared_runner_minutes_limit, ?, 0)', + current_application_settings.shared_runners_minutes) + + # this returns builds that are ordered by number of running builds + # we prefer projects that don't use shared runners at all + joins("LEFT JOIN (#{running_builds_for_shared_runners.to_sql}) AS project_builds ON ci_builds.gl_project_id=project_builds.gl_project_id"). + order('COALESCE(project_builds.running_builds, 0) ASC', 'ci_builds.id ASC') + end + + def builds_for_specific_runner + new_builds.where(project: current_runner.projects.with_builds_enabled).order('created_at ASC') + end + def running_builds_for_shared_runners Ci::Build.running.where(runner: Ci::Runner.shared). group(:gl_project_id).select(:gl_project_id, 'count(*) AS running_builds') end + + def new_builds + Ci::Build.pending.unstarted + end end end diff --git a/app/services/update_build_minutes_service.rb b/app/services/update_build_minutes_service.rb new file mode 100644 index 000000000000..47293cbf6e9c --- /dev/null +++ b/app/services/update_build_minutes_service.rb @@ -0,0 +1,10 @@ +class UpdateBuildMinutesService < BaseService + def execute(build) + return unless build.runner + return unless build.runner.shared? + return unless build.duration + + project.find_or_create_project_metrics. + update_all('shared_runners_minutes = shared_runners_minutes + ?', build.duration) + end +end diff --git a/app/views/admin/application_settings/_form.html.haml b/app/views/admin/application_settings/_form.html.haml index 95cae5ea24be..b475e7846052 100644 --- a/app/views/admin/application_settings/_form.html.haml +++ b/app/views/admin/application_settings/_form.html.haml @@ -196,6 +196,13 @@ = f.label :shared_runners_enabled do = f.check_box :shared_runners_enabled Enable shared runners for new projects + .form-group + = f.label :shared_runners_minutes, 'Shared runners minutes', class: 'control-label col-sm-2' + .col-sm-10 + = f.number_field :shared_runners_minutes, class: 'form-control' + .help-block + Set the maximum amount of minutes that project can use shared runners in period of time + = link_to "(?)", help_page_path("user/admin_area/settings/continuous_integration", anchor: "shared-runners-minutes") .form-group = f.label :shared_runners_text, class: 'control-label col-sm-2' .col-sm-10 diff --git a/app/views/admin/projects/show.html.haml b/app/views/admin/projects/show.html.haml index 6c7c3c48604a..b1952a2562b8 100644 --- a/app/views/admin/projects/show.html.haml +++ b/app/views/admin/projects/show.html.haml @@ -90,6 +90,20 @@ %span.light archived: %strong repository is read-only + - if @project.builds_enabled? + %li + %span.light Shared Runners: + %strong + - if @project.shared_runners_enabled? + Enabled + - if @project.shared_runner_minutes_limit.nonzero? + = @project.shared_runner_minutes_limit + total minutes + - elsif current_application_settings.shared_runners_minutes + Unlimited + - else + Disabled + %li %span.light access: %strong diff --git a/app/views/projects/builds/show.html.haml b/app/views/projects/builds/show.html.haml index 108674dbba66..661a392a048a 100644 --- a/app/views/projects/builds/show.html.haml +++ b/app/views/projects/builds/show.html.haml @@ -26,6 +26,19 @@ = link_to namespace_project_runners_path(@build.project.namespace, @build.project) do Runners page + - if @build.project.shared_runners_minutes_used? + .bs-callout.bs-callout-warning + %p + You did use all your allowed shared runners minutes: + = @build.project.shared_runners_minutes.to_i + of + = @build.project.shared_runners_minutes_limit + . + Consider looking at + = link_to namespace_project_runners_path(@build.project.namespace, @build.project) do + Runners page + . + - if @build.starts_environment? .prepend-top-default .environment-information diff --git a/app/workers/clear_shared_runner_minutes_worker.rb b/app/workers/clear_shared_runner_minutes_worker.rb new file mode 100644 index 000000000000..a62babc19d00 --- /dev/null +++ b/app/workers/clear_shared_runner_minutes_worker.rb @@ -0,0 +1,8 @@ +class ClearSharedRunnerMinutesWorker + include Sidekiq::Worker + include DedicatedSidekiqQueue + + def perform + ProjectMetrics.update_all(shared_runner_minutes: 0) + end +end diff --git a/config/initializers/1_settings.rb b/config/initializers/1_settings.rb index 9ddd15548111..3ccecc48d504 100644 --- a/config/initializers/1_settings.rb +++ b/config/initializers/1_settings.rb @@ -312,6 +312,10 @@ def host(url) Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['cron'] ||= '20 0 * * *' Settings.cron_jobs['remove_unreferenced_lfs_objects_worker']['job_class'] = 'RemoveUnreferencedLfsObjectsWorker' +Settings.cron_jobs['clear_shared_runner_minutes_worker'] ||= Settingslogic.new({}) +Settings.cron_jobs['clear_shared_runner_minutes_worker']['cron'] ||= '0 0 0 * *' +Settings.cron_jobs['clear_shared_runner_minutes_worker']['job_class'] = 'ClearSharedRunnerMinutesWorker' + # # GitLab Shell # diff --git a/db/migrate/20161129161815_add_shared_runners_minutes_to_application_settings.rb b/db/migrate/20161129161815_add_shared_runners_minutes_to_application_settings.rb new file mode 100644 index 000000000000..7d14f3a5ff7d --- /dev/null +++ b/db/migrate/20161129161815_add_shared_runners_minutes_to_application_settings.rb @@ -0,0 +1,9 @@ +class AddSharedRunnersMinutesToApplicationSettings < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :application_settings, :shared_runners_minutes, :integer, null: false, default: 0 + end +end diff --git a/db/migrate/20161129161913_add_shared_runners_minutes_limit_to_projects.rb b/db/migrate/20161129161913_add_shared_runners_minutes_limit_to_projects.rb new file mode 100644 index 000000000000..b33c5618b913 --- /dev/null +++ b/db/migrate/20161129161913_add_shared_runners_minutes_limit_to_projects.rb @@ -0,0 +1,9 @@ +class AddSharedRunnersMinutesLimitToProjects < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + def change + add_column :projects, :shared_runners_minutes_limit, :integer + end +end diff --git a/db/migrate/20161129161957_create_table_project_metrics.rb b/db/migrate/20161129161957_create_table_project_metrics.rb new file mode 100644 index 000000000000..21aeec74fe90 --- /dev/null +++ b/db/migrate/20161129161957_create_table_project_metrics.rb @@ -0,0 +1,18 @@ +# See http://doc.gitlab.com/ce/development/migration_style_guide.html +# for more information on how to write migrations for GitLab. + +class CreateTableProjectMetrics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + # Set this constant to true if this migration requires downtime. + DOWNTIME = false + + def change + create_table :project_metrics do |t| + t.integer :project_id, null: false + t.integer :shared_runners_minutes, default: 0, null: false + end + + add_foreign_key :project_metrics, :projects, column: :project_id, on_delete: :cascade + end +end diff --git a/db/migrate/20161129162216_add_index_to_project_metrics.rb b/db/migrate/20161129162216_add_index_to_project_metrics.rb new file mode 100644 index 000000000000..87ce5e334cf1 --- /dev/null +++ b/db/migrate/20161129162216_add_index_to_project_metrics.rb @@ -0,0 +1,11 @@ +class AddIndexToProjectMetrics < ActiveRecord::Migration + include Gitlab::Database::MigrationHelpers + + DOWNTIME = false + + disable_ddl_transaction! + + def change + add_concurrent_index :project_metrics, [:project_id], { unique: true } + end +end diff --git a/lib/api/entities.rb b/lib/api/entities.rb index 33cb6fd37040..70667c7bb609 100644 --- a/lib/api/entities.rb +++ b/lib/api/entities.rb @@ -101,6 +101,7 @@ class Project < Grape::Entity expose :only_allow_merge_if_build_succeeds expose :request_access_enabled expose :only_allow_merge_if_all_discussions_are_resolved + expose :shared_runners_minutes_limit end class Member < UserBasic -- GitLab