diff --git a/lib/gitlab_git.rb b/lib/gitlab_git.rb index 89bd83ba7120e2470191ca872072601fcb5eb98d..39a9d4b1509f8bed508497009a65a52d16fb520b 100644 --- a/lib/gitlab_git.rb +++ b/lib/gitlab_git.rb @@ -17,9 +17,11 @@ require_relative "gitlab_git/commit" require_relative "gitlab_git/commit_stats" require_relative "gitlab_git/compare" require_relative "gitlab_git/diff" +require_relative "gitlab_git/diff_collection" require_relative "gitlab_git/repository" require_relative "gitlab_git/tree" require_relative "gitlab_git/blob_snippet" require_relative "gitlab_git/ref" require_relative "gitlab_git/branch" require_relative "gitlab_git/tag" +require_relative "gitlab_git/util" diff --git a/lib/gitlab_git/commit.rb b/lib/gitlab_git/commit.rb index 47edd1438e17ca8b81943dd6c69e6f8424ee4672..bedf82f40ca41b40e74fdd0df6c0a02a56321bc6 100644 --- a/lib/gitlab_git/commit.rb +++ b/lib/gitlab_git/commit.rb @@ -205,7 +205,7 @@ module Gitlab end def diffs(options = {}) - diff_from_parent(options).map { |diff| Gitlab::Git::Diff.new(diff) } + DiffCollection.new(diff_from_parent(options), options) end def parents diff --git a/lib/gitlab_git/compare.rb b/lib/gitlab_git/compare.rb index 0c52c01f2e7e47bc76388040d3ec17f2856257ce..0e55fa190b3bd85034f5763bea610e2479b291b1 100644 --- a/lib/gitlab_git/compare.rb +++ b/lib/gitlab_git/compare.rb @@ -1,13 +1,12 @@ module Gitlab module Git class Compare - attr_reader :commits, :diffs, :same, :timeout, :head, :base + attr_reader :commits, :same, :head, :base def initialize(repository, base, head) - @commits, @diffs = [], [] + @commits = [] @same = false @repository = repository - @timeout = false return unless base && head @@ -24,30 +23,13 @@ module Gitlab @commits = Gitlab::Git::Commit.between(repository, @base.id, @head.id) end - def diffs(paths = nil, options = {}) + def diffs(options = {}) unless @head && @base - return [] + return Gitlab::Git::DiffCollection.new([]) end - # Try to collect diff only if diffs is empty - # Otherwise return cached version - if @diffs.empty? && @timeout == false - begin - @diffs = Gitlab::Git::Diff.between(@repository, @head.id, @base.id, - options, *paths) - rescue Gitlab::Git::Diff::TimeoutError => ex - @diffs = [] - @timeout = true - end - end - - @diffs - end - - # Check if diff is empty because it is actually empty - # and not because its impossible to get it - def empty_diff? - diffs.empty? && timeout == false + paths = options.delete(:paths) || [] + Gitlab::Git::Diff.between(@repository, @head.id, @base.id, options, *paths) end end end diff --git a/lib/gitlab_git/diff.rb b/lib/gitlab_git/diff.rb index 3879404f0a00180052ca100409281edb3db24f0f..781c75f11015db567bcaf780e04768dbb1d3c281 100644 --- a/lib/gitlab_git/diff.rb +++ b/lib/gitlab_git/diff.rb @@ -130,7 +130,8 @@ module Gitlab :disable_pathspec_match, :deltas_are_icase, :include_untracked_content, :skip_binary_check, :include_typechange, :include_typechange_trees, - :ignore_filemode, :recurse_ignored_dirs, :paths] + :ignore_filemode, :recurse_ignored_dirs, :paths, + :max_files, :max_lines, :all_diffs] if default_options actual_defaults = default_options.dup @@ -187,6 +188,10 @@ module Gitlab a_mode == '160000' || b_mode == '160000' end + def line_count + @line_count ||= Util.count_lines(@diff) + end + private def init_from_rugged(rugged) diff --git a/lib/gitlab_git/diff_collection.rb b/lib/gitlab_git/diff_collection.rb new file mode 100644 index 0000000000000000000000000000000000000000..1ebef8d0e048cf4df6db93bb5ccfe056db75d7c4 --- /dev/null +++ b/lib/gitlab_git/diff_collection.rb @@ -0,0 +1,90 @@ +module Gitlab + module Git + class DiffCollection + include Enumerable + + DEFAULT_LIMITS = { max_files: 100, max_lines: 5000 }.freeze + + def initialize(iterator, options={}) + @iterator = iterator + @max_files = options.fetch(:max_files, DEFAULT_LIMITS[:max_files]) + @max_lines = options.fetch(:max_lines, DEFAULT_LIMITS[:max_lines]) + @all_diffs = !!options.fetch(:all_diffs, false) + + @line_count = 0 + @overflow = false + @array = Array.new + end + + def each + @iterator.each_with_index do |raw, i| + # First yield cached Diff instances from @array + if @array[i] + yield @array[i] + next + end + + # We have exhausted @array, time to create new Diff instances or stop. + break if @overflow + + if !@all_diffs && i >= @max_files + @overflow = true + break + end + + # Going by the number of files alone it is OK to create a new Diff instance. + diff = Gitlab::Git::Diff.new(raw) + + @line_count += diff.line_count + if !@all_diffs && @line_count >= @max_lines + # This last Diff instance pushes us over the lines limit. We stop and + # discard it. + @overflow = true + break + end + + yield @array[i] = diff + end + end + + def empty? + !@iterator.any? + end + + def overflow? + populate! + !!@overflow + end + + def size + @size ||= count # forces a loop through @iterator + end + + def real_size + populate! + + if @overflow + "#{size}+" + else + size.to_s + end + end + + def decorate! + each_with_index do |element, i| + @array[i] = yield(element) + end + end + + private + + def populate! + return if @populated + + each { nil } # force a loop through all diffs + @populated = true + nil + end + end + end +end diff --git a/lib/gitlab_git/repository.rb b/lib/gitlab_git/repository.rb index 250ee83d03ae5c105a42a16002754de5719bbf91..cf87b4a27de1d436ffeadfa0a8682feffef10e4b 100644 --- a/lib/gitlab_git/repository.rb +++ b/lib/gitlab_git/repository.rb @@ -320,9 +320,7 @@ module Gitlab # diff options. The +options+ hash can also include :break_rewrites to # split larger rewrites into delete/add pairs. def diff(from, to, options = {}, *paths) - diff_patches(from, to, options, *paths).map do |p| - Gitlab::Git::Diff.new(p) - end + DiffCollection.new(diff_patches(from, to, options, *paths), options) end # Return the diff between +from+ and +to+ in a single patch string. The diff --git a/lib/gitlab_git/util.rb b/lib/gitlab_git/util.rb new file mode 100644 index 0000000000000000000000000000000000000000..03996b89d96b6fad199a836a4f07e662ea5d6608 --- /dev/null +++ b/lib/gitlab_git/util.rb @@ -0,0 +1,18 @@ +module Gitlab + module Git + module Util + LINE_SEP = "\n" + + def self.count_lines(string) + case string[-1] + when nil + 0 + when LINE_SEP + string.count(LINE_SEP) + else + string.count(LINE_SEP) + 1 + end + end + end + end +end diff --git a/spec/commit_spec.rb b/spec/commit_spec.rb index 83c718689785efbc736d1e5ec58f2590db24b1da..6a62fa89763dfee107bb0cb2e8750c16a7d66dc9 100644 --- a/spec/commit_spec.rb +++ b/spec/commit_spec.rb @@ -320,9 +320,9 @@ describe Gitlab::Git::Commit do describe :diffs do subject { commit.diffs } - it { should be_kind_of Array } - its(:size) { should eq(2) } - its(:first) { should be_kind_of Gitlab::Git::Diff } + it { should be_kind_of Gitlab::Git::DiffCollection } + it { subject.count.should eq(2) } + it { subject.first.should be_kind_of Gitlab::Git::Diff } end describe :ref_names do diff --git a/spec/compare_spec.rb b/spec/compare_spec.rb index 12ab017b20e82d8c8a94992d05dca5000c6f9654..88c50d415aa603fc51c9dffe2b57545bbb165097 100644 --- a/spec/compare_spec.rb +++ b/spec/compare_spec.rb @@ -22,8 +22,6 @@ describe Gitlab::Git::Compare do it { should have(10).elements } it { should include('files/ruby/popen.rb') } it { should_not include('LICENSE') } - it { compare.timeout.should be_false } - it { compare.empty_diff?.should be_false } end describe 'non-existing refs' do diff --git a/spec/diff_collection_spec.rb b/spec/diff_collection_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..20ffa4d8d2ec6dc80ba7ae27e495e1242cbf5e5e --- /dev/null +++ b/spec/diff_collection_spec.rb @@ -0,0 +1,138 @@ +require 'spec_helper' + +describe Gitlab::Git::DiffCollection do + subject do + Gitlab::Git::DiffCollection.new( + iterator, + max_files: max_files, + max_lines: max_lines, + all_diffs: all_diffs, + ) + end + let(:iterator) { Array.new(file_count, fake_diff(line_count)) } + let(:file_count) { 0 } + let(:line_count) { 1 } + let(:max_files) { 10 } + let(:max_lines) { 100 } + let(:all_diffs) { false } + + its(:to_a) { should be_kind_of ::Array } + + describe :decorate! do + let(:file_count) { 3} + + it 'modifies the array in place' do + count = 0 + subject.decorate! { |d| !d.nil? && count += 1 } + subject.to_a.should eq([1, 2, 3]) + end + end + + context 'overflow handling' do + context 'adding few enough files' do + let(:file_count) { 3 } + + context 'and few enough lines' do + let(:line_count) { 10 } + + its(:overflow?) { should be_false } + its(:empty?) { should be_false } + its(:real_size) { should eq('3') } + it { subject.size.should eq(3) } + + context 'when limiting is disabled' do + let(:all_diffs) { true } + + its(:overflow?) { should be_false } + its(:empty?) { should be_false } + its(:real_size) { should eq('3') } + it { subject.size.should eq(3) } + end + end + + context 'and too many lines' do + let(:line_count) { 1000 } + + its(:overflow?) { should be_true } + its(:empty?) { should be_false } + its(:real_size) { should eq('0+') } + it { subject.size.should eq(0) } + + context 'when limiting is disabled' do + let(:all_diffs) { true } + + its(:overflow?) { should be_false } + its(:empty?) { should be_false } + its(:real_size) { should eq('3') } + it { subject.size.should eq(3) } + end + end + end + + context 'adding too many files' do + let(:file_count) { 11 } + + context 'and few enough lines' do + let(:line_count) { 1 } + + its(:overflow?) { should be_true } + its(:empty?) { should be_false } + its(:real_size) { should eq('10+') } + it { subject.size.should eq(10) } + + context 'when limiting is disabled' do + let(:all_diffs) { true } + + its(:overflow?) { should be_false } + its(:empty?) { should be_false } + its(:real_size) { should eq('11') } + it { subject.size.should eq(11) } + end + end + + context 'and too many lines' do + let(:line_count) { 30 } + + its(:overflow?) { should be_true } + its(:empty?) { should be_false } + its(:real_size) { should eq('3+') } + it { subject.size.should eq(3) } + + context 'when limiting is disabled' do + let(:all_diffs) { true } + + its(:overflow?) { should be_false } + its(:empty?) { should be_false } + its(:real_size) { should eq('11') } + it { subject.size.should eq(11) } + end + end + end + + context 'adding exactly the maximum number of files' do + let(:file_count) { 10 } + + context 'and few enough lines' do + let(:line_count) { 1 } + + its(:overflow?) { should be_false } + its(:empty?) { should be_false } + its(:real_size) { should eq('10') } + it { subject.size.should eq(10) } + end + end + end + + describe 'empty collection' do + subject { Gitlab::Git::DiffCollection.new([]) } + + its(:overflow?) { should be_false } + its(:empty?) { should be_true } + its(:size) { should eq(0) } + its(:real_size) { should eq('0')} + end + + def fake_diff(line_count) + {'diff' => "DIFF\n" * line_count} + end +end diff --git a/spec/diff_spec.rb b/spec/diff_spec.rb index 46419f36c58a57ff0bbfb0e5780cbd5489755e9e..c15b539b7aade87e4dc985c3232c10afa47ad4df 100644 --- a/spec/diff_spec.rb +++ b/spec/diff_spec.rb @@ -52,7 +52,7 @@ EOT let(:diffs) { Gitlab::Git::Diff.between(repository, 'feature', 'master') } subject { diffs } - it { should be_kind_of Array } + it { should be_kind_of Gitlab::Git::DiffCollection } its(:size) { should eq(1) } context :diff do @@ -106,4 +106,10 @@ EOT it { Gitlab::Git::Diff.new(@diffs[0]).submodule?.should == false } it { Gitlab::Git::Diff.new(@diffs[1]).submodule?.should == true } end + + describe :line_count do + subject { Gitlab::Git::Diff.new(@rugged_diff) } + + its(:line_count) { should eq(9) } + end end diff --git a/spec/util_spec.rb b/spec/util_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..ea29c38ce3d07b41fc53eda460394d957324529c --- /dev/null +++ b/spec/util_spec.rb @@ -0,0 +1,16 @@ +require 'spec_helper' + +describe Gitlab::Git::Util do + describe :count_lines do + [ + ["", 0], + ["foo", 1], + ["foo\n", 1], + ["foo\n\n", 2], + ].each do |string, line_count| + it "counts #{line_count} lines in #{string.inspect}" do + Gitlab::Git::Util.count_lines(string).should eq(line_count) + end + end + end +end