Throughput chart MVC (BE)

Summary

This is the backend implementation issue for the Throughput Analytics feature.

Proposal

See the proposal in the implementation issue.

This MVC feature is at the project level.

Proof of concept

Please see the proof of concept that was created, built on the GitLab API:

throughput_by_month.rb

``` require 'date' require 'json' require 'net/http'

Purpose

Get data on throughput (number of merged merge requests).

Instructions

Customize access_token and query_params in the initialize method.

class ThroughputByMonth

attr_accessor(
  :access_token,
  :missing_merged_at_merge_request_iids,
  :query_params,
)

GITLAB_BOT_USER_ID = 1786152
GITLAB_ORG_GROUP_ID = 9970 # gitlab-org

def initialize
  self.access_token = 'PAT_GOES_HERE'
  self.missing_merged_at_merge_request_iids = []
  self.query_params = {
    # Documentation at https://docs.gitlab.com/ee/api/merge_requests.html#list-merge-requests
    # author_username: 'djensen',
    # created_after: '2020-01-01', # for testing only
    labels: 'group::analytics,frontend',
  }
end

def data
  {
    average_throughput: average_throughput_per_month,
    by_yearmonth: metrics_by_yearmonth.map do |yearmonth, metrics|
      [
        yearmonth,
        {
          count: metrics[:merged_merge_request_count],
          mttm_days: mttm_days(metrics),
        }
      ]
    end.to_h
  }
end

private

  def average_throughput_per_month
    throughputs = metrics_by_yearmonth.map { |yearmonth, metrics| metrics[:merged_merge_request_count] }
    total_throughput = throughputs.sum
    month_count = metrics_by_yearmonth.keys.size
    (total_throughput.to_f / month_count).round(1)
  end

  def build_query_string(params)
    query_string_default_params.merge(params).map do |key, value|
      "#{key}=#{value}"
    end.join('&')
  end

  def current_month_start_date
    @current_month_start_date ||= (Date.today - Date.today.mday + 1)
  end

  def time_from_string(string)
    DateTime.parse(string).to_time
  end

  def earliest_allowed_merged_at
    @earliest_allowed_merged_at ||= (current_month_start_date - 365).to_time
  end

  def get_all_response_results(url, params)
    results = []
    target_page = 1

    loop do
      params[:page] = target_page
      response = get_response(url, params)
      results.concat(JSON.parse(response.body))
      break if target_page >= response.header['X-Total-Pages'].to_i # Documented at https://docs.gitlab.com/ee/api/README.html#other-pagination-headers

      target_page += 1
    end

    return results
  end

  def get_response(url, params)
    sleep(0.1)
    query_string = build_query_string(params)
    uri = URI("#{url}?#{query_string}")
    request = Net::HTTP::Get.new(uri)
    Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http|
      http.request(request)
    end
  end

  def group_merge_requests_url
    @group_merge_requests_url ||= "https://gitlab.com/api/v4/groups/#{GITLAB_ORG_GROUP_ID}/merge_requests"
  end

  def increment_metrics_for_merge_request(merge_request:, metrics_by_yearmonth:)
    created_at = time_from_string(merge_request['created_at'])
    merged_at = merge_request_merged_at(merge_request)
    return if merged_at < earliest_allowed_merged_at
    # return if merged_at > current_month_start_date

    merged_yearmonth = yearmonth(merged_at)
    metrics_by_yearmonth[merged_yearmonth] = metrics_default unless metrics_by_yearmonth.has_key?(merged_yearmonth)
    metrics_by_yearmonth[merged_yearmonth][:merged_merge_request_count] += 1
    metrics_by_yearmonth[merged_yearmonth][:merge_request_iids] << merge_request['iid']
    metrics_by_yearmonth[merged_yearmonth][:total_time_to_merge] += (merged_at - created_at)
  end

  def last_complete_yearmonth
    @last_complete_yearmonth ||= team_metrics_by_yearmonth.sort.to_h.keys.last
  end

  def merge_request_merged_at(merge_request)
    merged_at = if merge_request['merged_at']
      merge_request['merged_at']
    else # missing merged_at per https://gitlab.com/gitlab-org/gitlab/issues/26911
      self.missing_merged_at_merge_request_iids << merge_request['id']
      merge_request['updated_at'] # closest proxy
    end
    time_from_string(merged_at)
  end

  def merged_merge_requests
    params = query_params.merge({ state: 'merged' })
    get_all_response_results(group_merge_requests_url, params)
  end

  def metrics_by_yearmonth
    return @metrics_by_yearmonth if @metrics_by_yearmonth
    
    @metrics_by_yearmonth = {}
    merged_merge_requests.each do |merge_request|
      increment_metrics_for_merge_request(merge_request: merge_request, metrics_by_yearmonth: metrics_by_yearmonth)
    end
    @metrics_by_yearmonth = @metrics_by_yearmonth.sort.to_h
  end

  def metrics_default
    {
      # authored_merge_request_count: 0,
      merged_merge_request_count: 0,
      merge_request_iids: [],
      total_time_to_merge: 0,
    }
  end

  def mttm_days(metrics)
    mttm_seconds = metrics[:total_time_to_merge] / metrics[:merge_request_iids].size
    (mttm_seconds / seconds_per_day).round(1)
  end

  def query_string_default_params
    {
      per_page: 50, # Max is 100
      private_token: access_token,
    }
  end

  def seconds_per_day
    @seconds_per_day ||= 60 * 60 * 24
  end

  def yearmonth(datetime)
    date = datetime.to_date
    date.strftime('%Y%m').to_i
  end

end

throughput = ThroughputByMonth.new puts throughput.data

</details>