From e79bbf4bdb4238e93f9adc40027280871e61c91e Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Tue, 1 Sep 2020 05:25:23 +0000 Subject: [PATCH 01/10] Revert "Revert "Merge branch..."" This reverts commit 5c234b63207730f7ac9bedf66b402d597c271c71. --- doc/integration/elasticsearch.md | 5 +- ee/app/models/ee/application_setting.rb | 2 +- ...ic-groups-bronze-and-above-to-advanced.yml | 5 + ee/spec/models/application_setting_spec.rb | 2 +- ee/spec/requests/api/search_spec.rb | 160 +++++++++--------- .../services/search/global_service_spec.rb | 4 +- 6 files changed, 92 insertions(+), 86 deletions(-) create mode 100644 ee/changelogs/unreleased/197231-default-searches-for-public-groups-bronze-and-above-to-advanced.yml diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index f788c2ca5cca5f..fbcfe8fa08de1e 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -237,7 +237,10 @@ If you select `Limit namespaces and projects that can be indexed`, more options You can select namespaces and projects to index exclusively. Note that if the namespace is a group it will include any sub-groups and projects belonging to those sub-groups to be indexed as well. -Elasticsearch only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search will not provide a code or commit scope. This will be possible only in the scope of an indexed namespace. Currently there is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. +NOTE: **Note:** +Elasticsearch only provides cross-group [code, commit, comment, and wiki](../user/search/advanced_global_search.md#overview) global search for indexed namespaces. + +In this particular scenario where only a subset of namespaces are indexed, a global search will provide code, commit, comment, and wiki scopes which return values from the subset of indexed namespaces. If you would like to perform a search against a namespace that is not indexed, you can select the group or project in the search filter. Additionally, it is possible to perform a global "basic search" by appending `&basic_search=true` to the URL in the browser bar. This search will use other data sources (that is PostgreSQL data and Git data) and the code, commit, comment, and wiki scopes will not be displayed. You can filter the selection dropdown by writing part of the namespace or project name you're interested in. diff --git a/ee/app/models/ee/application_setting.rb b/ee/app/models/ee/application_setting.rb index 688e8a393a7c1c..27e52e71834c39 100644 --- a/ee/app/models/ee/application_setting.rb +++ b/ee/app/models/ee/application_setting.rb @@ -244,7 +244,7 @@ def search_using_elasticsearch?(scope: nil) when Project elasticsearch_indexes_project?(scope) else - false # Never use elasticsearch for the global scope when limiting is on + true # Use elasticsearch for the global scope, even when limiting is on end end diff --git a/ee/changelogs/unreleased/197231-default-searches-for-public-groups-bronze-and-above-to-advanced.yml b/ee/changelogs/unreleased/197231-default-searches-for-public-groups-bronze-and-above-to-advanced.yml new file mode 100644 index 00000000000000..d5dd38d742497e --- /dev/null +++ b/ee/changelogs/unreleased/197231-default-searches-for-public-groups-bronze-and-above-to-advanced.yml @@ -0,0 +1,5 @@ +--- +title: Enable Advanced Search for global scope searches +merge_request: 38093 +author: +type: changed diff --git a/ee/spec/models/application_setting_spec.rb b/ee/spec/models/application_setting_spec.rb index 2a387037cb5988..08922e848a2dae 100644 --- a/ee/spec/models/application_setting_spec.rb +++ b/ee/spec/models/application_setting_spec.rb @@ -435,7 +435,7 @@ def expect_is_es_licensed context 'global scope' do let(:scope) { nil } - it { is_expected.to eq(only_when_enabled_globally) } + it { is_expected.to eq(indexing && searching) } end context 'namespace (in scope)' do diff --git a/ee/spec/requests/api/search_spec.rb b/ee/spec/requests/api/search_spec.rb index ee44bebc040605..4b401f4e92c319 100644 --- a/ee/spec/requests/api/search_spec.rb +++ b/ee/spec/requests/api/search_spec.rb @@ -61,10 +61,7 @@ shared_examples 'elasticsearch enabled' do |level:| context 'for merge_requests scope', :sidekiq_inline do before do - create(:labeled_merge_request, target_branch: 'feature_1', source_project: project, labels: [create(:label), create(:label)]) - create(:merge_request, target_branch: 'feature_2', source_project: project, author: create(:user)) - create(:merge_request, target_branch: 'feature_3', source_project: project, milestone: create(:milestone, project: project)) - create(:merge_request, target_branch: 'feature_4', source_project: project) + create_list(:merge_request, 3, :unique_branches, source_project: project, author: create(:user), milestone: create(:milestone, project: project), labels: [create(:label)]) ensure_elasticsearch_index! end @@ -72,19 +69,14 @@ it 'avoids N+1 queries' do control = ActiveRecord::QueryRecorder.new { get api(endpoint, user), params: { scope: 'merge_requests', search: '*' } } - - create(:labeled_merge_request, target_branch: 'feature_5', source_project: project, labels: [create(:label), create(:label)]) - create(:merge_request, target_branch: 'feature_6', source_project: project, author: create(:user)) - create(:merge_request, target_branch: 'feature_7', source_project: project, milestone: create(:milestone, project: project)) - create(:merge_request, target_branch: 'feature_8', source_project: project) - + create_list(:merge_request, 3, :unique_branches, source_project: project, author: create(:user), milestone: create(:milestone, project: project), labels: [create(:label)]) ensure_elasticsearch_index! expect { get api(endpoint, user), params: { scope: 'merge_requests', search: '*' } }.not_to exceed_query_limit(control) end end - context 'for wiki_blobs scope', :sidekiq_might_not_need_inline do + context 'for wiki_blobs scope', :sidekiq_inline do before do wiki = create(:project_wiki, project: project) create(:wiki_page, wiki: wiki, title: 'home', content: "Awesome page") @@ -101,105 +93,105 @@ it_behaves_like 'pagination', scope: 'wiki_blobs' end - context 'for commits scope', :sidekiq_inline do + context 'for commits and blobs', :sidekiq_inline do before do project.repository.index_commits_and_blobs ensure_elasticsearch_index! - - get api(endpoint, user), params: { scope: 'commits', search: 'folder' } end - it_behaves_like 'response is correct', schema: 'public_api/v4/commits_details', size: 2 - - it_behaves_like 'pagination', scope: 'commits' + context 'for commits scope' do + before do + get api(endpoint, user), params: { scope: 'commits', search: 'folder' } + end - it 'avoids N+1 queries' do - control = ActiveRecord::QueryRecorder.new { get api(endpoint, user), params: { scope: 'commits', search: 'folder' } } + it_behaves_like 'response is correct', schema: 'public_api/v4/commits_details', size: 2 - project_2 = create(:project, :public, :repository, :wiki_repo, name: 'awesome project 2') - project_3 = create(:project, :public, :repository, :wiki_repo, name: 'awesome project 3') - project_2.repository.index_commits_and_blobs - project_3.repository.index_commits_and_blobs + it_behaves_like 'pagination', scope: 'commits' - ensure_elasticsearch_index! + it 'avoids N+1 queries' do + control = ActiveRecord::QueryRecorder.new { get api(endpoint, user), params: { scope: 'commits', search: 'folder' } } - # Some N+1 queries still exist - expect { get api(endpoint, user), params: { scope: 'commits', search: 'folder' } }.not_to exceed_query_limit(control).with_threshold(9) - end - end + project_2 = create(:project, :public, :repository, :wiki_repo, name: 'awesome project 2') + project_3 = create(:project, :public, :repository, :wiki_repo, name: 'awesome project 3') + project_2.repository.index_commits_and_blobs + project_3.repository.index_commits_and_blobs - context 'for blobs scope', :sidekiq_might_not_need_inline do - before do - project.repository.index_commits_and_blobs - ensure_elasticsearch_index! + ensure_elasticsearch_index! - get api(endpoint, user), params: { scope: 'blobs', search: 'monitors' } + # Some N+1 queries still exist + expect { get api(endpoint, user), params: { scope: 'commits', search: 'folder' } }.not_to exceed_query_limit(control).with_threshold(9) + end end - it_behaves_like 'response is correct', schema: 'public_api/v4/blobs' - - it_behaves_like 'pagination', scope: 'blobs' - - context 'filters' do - def results_filenames - json_response.map { |h| h['filename'] }.compact + context 'for blobs scope' do + before do + get api(endpoint, user), params: { scope: 'blobs', search: 'monitors' } end - def results_paths - json_response.map { |h| h['path'] }.compact - end + it_behaves_like 'response is correct', schema: 'public_api/v4/blobs' - context 'with an including filter' do - it 'by filename' do - get api("/projects/#{project.id}/search", user), params: { scope: 'blobs', search: 'mon* filename:PROCESS.md' } + it_behaves_like 'pagination', scope: 'blobs' - expect(response).to have_gitlab_http_status(:ok) - expect(json_response.size).to eq(1) - expect(results_filenames).to all(match(%r{PROCESS.md$})) + context 'filters' do + def results_filenames + json_response.map { |h| h['filename'] }.compact end - it 'by path' do - get api("/projects/#{project.id}/search", user), params: { scope: 'blobs', search: 'mon* path:files/markdown' } - - expect(response).to have_gitlab_http_status(:ok) - expect(json_response.size).to eq(1) - expect(results_paths).to all(match(%r{^files/markdown/})) + def results_paths + json_response.map { |h| h['path'] }.compact end - it 'by extension' do - get api("/projects/#{project.id}/search", user), params: { scope: 'blobs', search: 'mon* extension:md' } + context 'with an including filter' do + it 'by filename' do + get api("/projects/#{project.id}/search", user), params: { scope: 'blobs', search: 'mon* filename:PROCESS.md' } - expect(response).to have_gitlab_http_status(:ok) - expect(json_response.size).to eq(3) + expect(response).to have_gitlab_http_status(:ok) + expect(json_response.size).to eq(1) + expect(results_filenames).to all(match(%r{PROCESS.md$})) + end - expect(results_filenames).to all(match(%r{.*.md$})) - end - end + it 'by path' do + get api("/projects/#{project.id}/search", user), params: { scope: 'blobs', search: 'mon* path:markdown' } - context 'with an excluding filter' do - it 'by filename' do - get api(endpoint, user), params: { scope: 'blobs', search: '* -filename:PROCESS.md' } + expect(response).to have_gitlab_http_status(:ok) + expect(json_response.size).to eq(1) + expect(results_paths).to all(match(%r{^files/markdown/})) + end - expect(response).to have_gitlab_http_status(:ok) - expect(results_filenames).not_to include('PROCESS.md') - expect(json_response.size).to eq(20) + it 'by extension' do + get api("/projects/#{project.id}/search", user), params: { scope: 'blobs', search: 'mon* extension:md' } + + expect(response).to have_gitlab_http_status(:ok) + expect(json_response.size).to eq(3) + expect(results_filenames).to all(match(%r{.*.md$})) + end end - it 'by path' do - get api(endpoint, user), params: { scope: 'blobs', search: '* -path:files/markdown' } + context 'with an excluding filter' do + it 'by filename' do + get api(endpoint, user), params: { scope: 'blobs', search: '* -filename:PROCESS.md' } - expect(response).to have_gitlab_http_status(:ok) - expect(results_paths).not_to include(a_string_matching(%r{^files/markdown/})) - expect(json_response.size).to eq(20) - end + expect(response).to have_gitlab_http_status(:ok) + expect(results_filenames).not_to include('PROCESS.md') + expect(json_response.size).to eq(20) + end - it 'by extension' do - get api(endpoint, user), params: { scope: 'blobs', search: '* -extension:md' } + it 'by path' do + get api(endpoint, user), params: { scope: 'blobs', search: '* -path:files/markdown' } - expect(response).to have_gitlab_http_status(:ok) + expect(response).to have_gitlab_http_status(:ok) + expect(results_paths).not_to include(a_string_matching(%r{^files/markdown/})) + expect(json_response.size).to eq(20) + end + + it 'by extension' do + get api(endpoint, user), params: { scope: 'blobs', search: '* -extension:md' } - expect(results_filenames).not_to include(a_string_matching(%r{.*.md$})) - expect(json_response.size).to eq(20) + expect(response).to have_gitlab_http_status(:ok) + + expect(results_filenames).not_to include(a_string_matching(%r{.*.md$})) + expect(json_response.size).to eq(20) + end end end end @@ -270,7 +262,7 @@ def results_paths end end - context 'for users scope', :sidekiq_inline do + context 'for users scope', :sidekiq_might_not_need_inline do before do create_list(:user, 2).each do |user| project.add_developer(user) @@ -326,7 +318,13 @@ def results_paths stub_ee_application_setting(elasticsearch_limit_indexing: true) end - it_behaves_like 'elasticsearch disabled' + context 'and namespace is indexed' do + before do + create :elasticsearch_indexed_namespace, namespace: group + end + + it_behaves_like 'elasticsearch enabled', level: :global + end end context 'when elasticsearch_limit_indexing off' do diff --git a/ee/spec/services/search/global_service_spec.rb b/ee/spec/services/search/global_service_spec.rb index aa4541656080a7..e702f207507cfa 100644 --- a/ee/spec/services/search/global_service_spec.rb +++ b/ee/spec/services/search/global_service_spec.rb @@ -182,8 +182,8 @@ stub_ee_application_setting(elasticsearch_limit_indexing: true) end - it 'does not include ES-specific scopes' do - expect(described_class.new(user, {}).allowed_scopes).not_to include('commits') + it 'includes ES-specific scopes' do + expect(described_class.new(user, {}).allowed_scopes).to include('commits') end end end -- GitLab From b7d0d1104000c026e2aa38415e4eac2a866500cc Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 2 Sep 2020 03:55:53 +0000 Subject: [PATCH 02/10] Add feature flag advanced_global_search_for_limited_indexing This will allow us to easily enable/disable global searches going to Elasticsearch if we have another incident like https://gitlab.com/gitlab-com/gl-infra/production/-/issues/2499 . --- ee/app/models/ee/application_setting.rb | 2 +- ...ic-groups-bronze-and-above-to-advanced.yml | 5 ---- ...ced_global_search_for_limited_indexing.yml | 7 +++++ ee/spec/models/application_setting_spec.rb | 6 +++-- .../services/search/global_service_spec.rb | 27 ++++++++++++++++--- 5 files changed, 36 insertions(+), 11 deletions(-) delete mode 100644 ee/changelogs/unreleased/197231-default-searches-for-public-groups-bronze-and-above-to-advanced.yml create mode 100644 ee/config/feature_flags/development/advanced_global_search_for_limited_indexing.yml diff --git a/ee/app/models/ee/application_setting.rb b/ee/app/models/ee/application_setting.rb index 27e52e71834c39..d3b96e64048943 100644 --- a/ee/app/models/ee/application_setting.rb +++ b/ee/app/models/ee/application_setting.rb @@ -244,7 +244,7 @@ def search_using_elasticsearch?(scope: nil) when Project elasticsearch_indexes_project?(scope) else - true # Use elasticsearch for the global scope, even when limiting is on + ::Feature.enabled?(:advanced_global_search_for_limited_indexing) end end diff --git a/ee/changelogs/unreleased/197231-default-searches-for-public-groups-bronze-and-above-to-advanced.yml b/ee/changelogs/unreleased/197231-default-searches-for-public-groups-bronze-and-above-to-advanced.yml deleted file mode 100644 index d5dd38d742497e..00000000000000 --- a/ee/changelogs/unreleased/197231-default-searches-for-public-groups-bronze-and-above-to-advanced.yml +++ /dev/null @@ -1,5 +0,0 @@ ---- -title: Enable Advanced Search for global scope searches -merge_request: 38093 -author: -type: changed diff --git a/ee/config/feature_flags/development/advanced_global_search_for_limited_indexing.yml b/ee/config/feature_flags/development/advanced_global_search_for_limited_indexing.yml new file mode 100644 index 00000000000000..1b4977fe6cfd51 --- /dev/null +++ b/ee/config/feature_flags/development/advanced_global_search_for_limited_indexing.yml @@ -0,0 +1,7 @@ +--- +name: advanced_global_search_for_limited_indexing +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41041 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/244276 +group: group::global search +type: development +default_enabled: false \ No newline at end of file diff --git a/ee/spec/models/application_setting_spec.rb b/ee/spec/models/application_setting_spec.rb index 08922e848a2dae..9091f60d6135ab 100644 --- a/ee/spec/models/application_setting_spec.rb +++ b/ee/spec/models/application_setting_spec.rb @@ -408,7 +408,7 @@ def expect_is_es_licensed describe '#search_using_elasticsearch?' do # Constructs a truth table with 16 entries to run the specs against - where(indexing: [true, false], searching: [true, false], limiting: [true, false]) + where(indexing: [true, false], searching: [true, false], limiting: [true, false], advanced_global_search_for_limited_indexing: [true, false]) with_them do let_it_be(:included_project_container) { create(:elasticsearch_indexed_project) } @@ -430,12 +430,14 @@ def expect_is_es_licensed elasticsearch_search: searching, elasticsearch_limit_indexing: limiting ) + + stub_feature_flags(advanced_global_search_for_limited_indexing: advanced_global_search_for_limited_indexing) end context 'global scope' do let(:scope) { nil } - it { is_expected.to eq(indexing && searching) } + it { is_expected.to eq(indexing && searching && (!limiting || advanced_global_search_for_limited_indexing)) } end context 'namespace (in scope)' do diff --git a/ee/spec/services/search/global_service_spec.rb b/ee/spec/services/search/global_service_spec.rb index e702f207507cfa..d7649ca65929ca 100644 --- a/ee/spec/services/search/global_service_spec.rb +++ b/ee/spec/services/search/global_service_spec.rb @@ -177,13 +177,34 @@ end end - context 'when ES is not used' do + context 'when elasticearch_search is disabled' do + before do + stub_ee_application_setting(elasticsearch_search: false) + end + + it 'does not include ES-specific scopes' do + expect(described_class.new(user, {}).allowed_scopes).not_to include('commits') + end + end + + context 'when elasticsearch_limit_indexing is enabled' do before do stub_ee_application_setting(elasticsearch_limit_indexing: true) + stub_feature_flags(advanced_global_search_for_limited_indexing: false) end - it 'includes ES-specific scopes' do - expect(described_class.new(user, {}).allowed_scopes).to include('commits') + it 'does not include ES-specific scopes' do + expect(described_class.new(user, {}).allowed_scopes).not_to include('commits') + end + + context 'when advanced_global_search_for_limited_indexing feature flag is enabled' do + before do + stub_feature_flags(advanced_global_search_for_limited_indexing: true) + end + + it 'includes ES-specific scopes' do + expect(described_class.new(user, {}).allowed_scopes).to include('commits') + end end end end -- GitLab From 444906c1ff4ef44a09b8e929b04a936498085805 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 2 Sep 2020 06:13:55 +0000 Subject: [PATCH 03/10] Add block_anonymous_global_searches feature flag This will allow us to disable anonymous global searches which will help with managing the impact of defaulting global searches to using Elasticsearch. Searching via the API is already blocked for unauthenticed users so it was only necessary to add this check to the `SearchController`. --- app/controllers/search_controller.rb | 9 ++++ .../block_anonymous_global_searches.yml | 7 +++ .../search/elastic/global_search_spec.rb | 20 +++++++- .../search/elastic/snippet_search_spec.rb | 32 ++++++++---- .../redacted_search_results_examples.rb | 4 +- locale/gitlab.pot | 3 ++ spec/controllers/search_controller_spec.rb | 30 ++++++++++++ .../search/user_searches_for_issues_spec.rb | 34 +++++++++---- .../search/user_searches_for_projects_spec.rb | 49 +++++++++++++------ spec/frontend/fixtures/search.rb | 6 +++ 10 files changed, 154 insertions(+), 40 deletions(-) create mode 100644 config/feature_flags/development/block_anonymous_global_searches.yml diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index db3c90007bca2b..fcb4c2bd8d70be 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -15,6 +15,7 @@ class SearchController < ApplicationController around_action :allow_gitaly_ref_name_caching + before_action :block_anonymous_global_searches, if: -> { ::Feature.enabled?(:block_anonymous_global_searches) } skip_before_action :authenticate_user! requires_cross_project_access if: -> do search_term_present = params[:search].present? || params[:term].present? @@ -128,6 +129,14 @@ def append_info_to_payload(payload) payload[:metadata]['meta.search.search'] = params[:search] payload[:metadata]['meta.search.scope'] = params[:scope] end + + def block_anonymous_global_searches + if params[:project_id].blank? && params[:group_id].blank? && !current_user + store_location_for(:user, request.fullpath) + + redirect_to new_user_session_path, alert: _('You must be logged in to search across all of GitLab') + end + end end SearchController.prepend_if_ee('EE::SearchController') diff --git a/config/feature_flags/development/block_anonymous_global_searches.yml b/config/feature_flags/development/block_anonymous_global_searches.yml new file mode 100644 index 00000000000000..527e99ad8a8b5e --- /dev/null +++ b/config/feature_flags/development/block_anonymous_global_searches.yml @@ -0,0 +1,7 @@ +--- +name: block_anonymous_global_searches +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/41041 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/244276 +group: group::global search +type: development +default_enabled: false \ No newline at end of file diff --git a/ee/spec/features/search/elastic/global_search_spec.rb b/ee/spec/features/search/elastic/global_search_spec.rb index e8db327e415996..5a484f58b74fa6 100644 --- a/ee/spec/features/search/elastic/global_search_spec.rb +++ b/ee/spec/features/search/elastic/global_search_spec.rb @@ -266,7 +266,23 @@ end RSpec.describe 'Global elastic search redactions', :elastic do - it_behaves_like 'a redacted search results page' do - let(:search_path) { explore_root_path } + context 'when block_anonymous_global_searches is disabled' do + before do + stub_feature_flags(block_anonymous_global_searches: false) + end + + it_behaves_like 'a redacted search results page' do + let(:search_path) { explore_root_path } + end + end + + context 'when block_anonymous_global_searches is enabled' do + before do + stub_feature_flags(block_anonymous_global_searches: true) + end + + it_behaves_like 'a redacted search results page', include_anonymous: false do + let(:search_path) { explore_root_path } + end end end diff --git a/ee/spec/features/search/elastic/snippet_search_spec.rb b/ee/spec/features/search/elastic/snippet_search_spec.rb index 42c750849d8e56..ae6d3ad9449d2c 100644 --- a/ee/spec/features/search/elastic/snippet_search_spec.rb +++ b/ee/spec/features/search/elastic/snippet_search_spec.rb @@ -37,19 +37,31 @@ context 'as anonymous user' do let(:current_user) { nil } - it 'finds only public snippets' do - within('.results') do - expect(page).to have_content('public personal snippet') - expect(page).not_to have_content('public project snippet') + context 'when block_anonymous_global_searches is enabled' do + it 'redirects to login page' do + expect(page).to have_content('You must be logged in to search across all of GitLab') + end + end - expect(page).not_to have_content('internal personal snippet') - expect(page).not_to have_content('internal project snippet') + context 'when block_anonymous_global_searches is disabled' do + before(:context) do + stub_feature_flags(block_anonymous_global_searches: false) + end - expect(page).not_to have_content('authorized personal snippet') - expect(page).not_to have_content('authorized project snippet') + it 'finds only public snippets' do + within('.results') do + expect(page).to have_content('public personal snippet') + expect(page).not_to have_content('public project snippet') - expect(page).not_to have_content('private personal snippet') - expect(page).not_to have_content('private project snippet') + expect(page).not_to have_content('internal personal snippet') + expect(page).not_to have_content('internal project snippet') + + expect(page).not_to have_content('authorized personal snippet') + expect(page).not_to have_content('authorized project snippet') + + expect(page).not_to have_content('private personal snippet') + expect(page).not_to have_content('private project snippet') + end end end end diff --git a/ee/spec/support/features/redacted_search_results_examples.rb b/ee/spec/support/features/redacted_search_results_examples.rb index 67c5501fc8e45e..20313640351801 100644 --- a/ee/spec/support/features/redacted_search_results_examples.rb +++ b/ee/spec/support/features/redacted_search_results_examples.rb @@ -1,6 +1,6 @@ # frozen_string_literal: true -RSpec.shared_examples 'a redacted search results page' do +RSpec.shared_examples 'a redacted search results page' do |include_anonymous: true| let(:public_group) { create(:group, :public) } let(:public_restricted_project) { create(:project, :repository, :public, :wiki_repo, namespace: public_group, name: 'The Project') } let(:issue_access_level) { ProjectFeature::PRIVATE } @@ -41,7 +41,7 @@ end it_behaves_like 'redacted search results page assertions', true - it_behaves_like 'redacted search results page assertions', false + it_behaves_like 'redacted search results page assertions', false if include_anonymous end # Only intended to be used in the above shared examples to avoid duplication of diff --git a/locale/gitlab.pot b/locale/gitlab.pot index a43e40daecf303..dde50f45f33614 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -28853,6 +28853,9 @@ msgstr "" msgid "You must accept our Terms of Service and privacy policy in order to register an account" msgstr "" +msgid "You must be logged in to search across all of GitLab" +msgstr "" + msgid "You must disassociate %{domain} from all clusters it is attached to before deletion." msgstr "" diff --git a/spec/controllers/search_controller_spec.rb b/spec/controllers/search_controller_spec.rb index e85330b08f6c66..3d883e076a6068 100644 --- a/spec/controllers/search_controller_spec.rb +++ b/spec/controllers/search_controller_spec.rb @@ -95,6 +95,10 @@ using RSpec::Parameterized::TableSyntax render_views + before do + stub_feature_flags(block_anonymous_global_searches: false) + end + it 'omits pipeline status from load' do project = create(:project, :public) expect(Gitlab::Cache::Ci::ProjectPipelineStatus).not_to receive(:load_in_batch_for_projects) @@ -138,6 +142,32 @@ end end end + + context 'when block_anonymous_global_searches is enabled' do + before do + stub_feature_flags(block_anonymous_global_searches: true) + end + + context 'for unauthenticated user' do + before do + sign_out(user) + end + + it 'redirects to login page' do + get :show, params: { scope: 'projects', search: '*' } + + expect(response).to redirect_to new_user_session_path + end + end + + context 'for authenticated user' do + it 'succeeds' do + get :show, params: { scope: 'projects', search: '*' } + + expect(response).to have_gitlab_http_status(:ok) + end + end + end end it 'finds issue comments' do diff --git a/spec/features/search/user_searches_for_issues_spec.rb b/spec/features/search/user_searches_for_issues_spec.rb index d895e87f1cb599..e6f05f9d72d463 100644 --- a/spec/features/search/user_searches_for_issues_spec.rb +++ b/spec/features/search/user_searches_for_issues_spec.rb @@ -86,20 +86,34 @@ def search_for_issue(search) end context 'when signed out' do - let(:project) { create(:project, :public) } + context 'when block_anonymous_global_searches is disabled' do + let(:project) { create(:project, :public) } - before do - visit(search_path) - end + before do + stub_feature_flags(block_anonymous_global_searches: false) + visit(search_path) + end - include_examples 'top right search form' + include_examples 'top right search form' - it 'finds an issue' do - search_for_issue(issue1.title) + it 'finds an issue' do + search_for_issue(issue1.title) - page.within('.results') do - expect(page).to have_link(issue1.title) - expect(page).not_to have_link(issue2.title) + page.within('.results') do + expect(page).to have_link(issue1.title) + expect(page).not_to have_link(issue2.title) + end + end + end + + context 'when block_anonymous_global_searches is enabled' do + before do + stub_feature_flags(block_anonymous_global_searches: true) + visit(search_path) + end + + it 'is redirected to login page' do + expect(page).to have_content('You must be logged in to search across all of GitLab') end end end diff --git a/spec/features/search/user_searches_for_projects_spec.rb b/spec/features/search/user_searches_for_projects_spec.rb index 7bb5a4da7d0895..6116d8c4c874fc 100644 --- a/spec/features/search/user_searches_for_projects_spec.rb +++ b/spec/features/search/user_searches_for_projects_spec.rb @@ -6,31 +6,48 @@ let!(:project) { create(:project, :public, name: 'Shop') } context 'when signed out' do - include_examples 'top right search form' + context 'when block_anonymous_global_searches is disabled' do + before do + stub_feature_flags(block_anonymous_global_searches: false) + end - it 'finds a project' do - visit(search_path) + include_examples 'top right search form' - fill_in('dashboard_search', with: project.name[0..3]) - click_button('Search') + it 'finds a project' do + visit(search_path) - expect(page).to have_link(project.name) - end + fill_in('dashboard_search', with: project.name[0..3]) + click_button('Search') - it 'preserves the group being searched in' do - visit(search_path(group_id: project.namespace.id)) + expect(page).to have_link(project.name) + end - submit_search('foo') + it 'preserves the group being searched in' do + visit(search_path(group_id: project.namespace.id)) - expect(find('#group_id', visible: false).value).to eq(project.namespace.id.to_s) - end + submit_search('foo') + + expect(find('#group_id', visible: false).value).to eq(project.namespace.id.to_s) + end - it 'preserves the project being searched in' do - visit(search_path(project_id: project.id)) + it 'preserves the project being searched in' do + visit(search_path(project_id: project.id)) + + submit_search('foo') + + expect(find('#project_id', visible: false).value).to eq(project.id.to_s) + end + end - submit_search('foo') + context 'when block_anonymous_global_searches is enabled' do + before do + stub_feature_flags(block_anonymous_global_searches: true) + end - expect(find('#project_id', visible: false).value).to eq(project.id.to_s) + it 'is redirected to login page' do + visit(search_path) + expect(page).to have_content('You must be logged in to search across all of GitLab') + end end end end diff --git a/spec/frontend/fixtures/search.rb b/spec/frontend/fixtures/search.rb index fcd68662acc17a..40f613a9422b6e 100644 --- a/spec/frontend/fixtures/search.rb +++ b/spec/frontend/fixtures/search.rb @@ -7,10 +7,16 @@ render_views + let_it_be(:user) { create(:admin) } + before(:all) do clean_frontend_fixtures('search/') end + before do + sign_in(user) + end + it 'search/show.html' do get :show -- GitLab From ba5a29b36d1c919bed23df20198e01e0812e93b9 Mon Sep 17 00:00:00 2001 From: Dylan Griffith Date: Wed, 2 Sep 2020 06:41:25 +0000 Subject: [PATCH 04/10] Remove the new advanced_global_search_for_limited_indexing docs We will add these back in after we remove the feature flag so that we cause less confusion as explaining both branches of logic will not be all that helpful and the feature flag should be cleaned up quite quickly when we can be sure this is stable on GitLab.com . --- doc/integration/elasticsearch.md | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/doc/integration/elasticsearch.md b/doc/integration/elasticsearch.md index fbcfe8fa08de1e..f788c2ca5cca5f 100644 --- a/doc/integration/elasticsearch.md +++ b/doc/integration/elasticsearch.md @@ -237,10 +237,7 @@ If you select `Limit namespaces and projects that can be indexed`, more options You can select namespaces and projects to index exclusively. Note that if the namespace is a group it will include any sub-groups and projects belonging to those sub-groups to be indexed as well. -NOTE: **Note:** -Elasticsearch only provides cross-group [code, commit, comment, and wiki](../user/search/advanced_global_search.md#overview) global search for indexed namespaces. - -In this particular scenario where only a subset of namespaces are indexed, a global search will provide code, commit, comment, and wiki scopes which return values from the subset of indexed namespaces. If you would like to perform a search against a namespace that is not indexed, you can select the group or project in the search filter. Additionally, it is possible to perform a global "basic search" by appending `&basic_search=true` to the URL in the browser bar. This search will use other data sources (that is PostgreSQL data and Git data) and the code, commit, comment, and wiki scopes will not be displayed. +Elasticsearch only provides cross-group code/commit search (global) if all name-spaces are indexed. In this particular scenario where only a subset of namespaces are indexed, a global search will not provide a code or commit scope. This will be possible only in the scope of an indexed namespace. Currently there is no way to code/commit search in multiple indexed namespaces (when only a subset of namespaces has been indexed). For example if two groups are indexed, there is no way to run a single code search on both. You can only run a code search on the first group and then on the second. You can filter the selection dropdown by writing part of the namespace or project name you're interested in. -- GitLab From 97f942bbea46ad477c74e6b362eee2ab1d1bbb5f Mon Sep 17 00:00:00 2001 From: Terri Chu Date: Thu, 10 Sep 2020 00:29:35 +0000 Subject: [PATCH 05/10] Apply 1 suggestion(s) to 1 file(s) --- ee/spec/models/application_setting_spec.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ee/spec/models/application_setting_spec.rb b/ee/spec/models/application_setting_spec.rb index 9091f60d6135ab..e451aba5d15903 100644 --- a/ee/spec/models/application_setting_spec.rb +++ b/ee/spec/models/application_setting_spec.rb @@ -407,7 +407,7 @@ def expect_is_es_licensed end describe '#search_using_elasticsearch?' do - # Constructs a truth table with 16 entries to run the specs against + # Constructs a truth table to run the specs against where(indexing: [true, false], searching: [true, false], limiting: [true, false], advanced_global_search_for_limited_indexing: [true, false]) with_them do -- GitLab From 87a7fa0ff4940827f67953ee5f347e8da5d38225 Mon Sep 17 00:00:00 2001 From: Terri Chu Date: Wed, 9 Sep 2020 20:31:52 -0400 Subject: [PATCH 06/10] fixup! Revert "Revert "Merge branch..."" --- app/controllers/search_controller.rb | 4 ++-- ee/spec/features/search/elastic/global_search_spec.rb | 4 ---- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index fcb4c2bd8d70be..5b623a4fb93164 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -15,7 +15,7 @@ class SearchController < ApplicationController around_action :allow_gitaly_ref_name_caching - before_action :block_anonymous_global_searches, if: -> { ::Feature.enabled?(:block_anonymous_global_searches) } + before_action :block_anonymous_global_searches skip_before_action :authenticate_user! requires_cross_project_access if: -> do search_term_present = params[:search].present? || params[:term].present? @@ -131,7 +131,7 @@ def append_info_to_payload(payload) end def block_anonymous_global_searches - if params[:project_id].blank? && params[:group_id].blank? && !current_user + if params[:project_id].blank? && params[:group_id].blank? && !current_user && ::Feature.enabled?(:block_anonymous_global_searches) store_location_for(:user, request.fullpath) redirect_to new_user_session_path, alert: _('You must be logged in to search across all of GitLab') diff --git a/ee/spec/features/search/elastic/global_search_spec.rb b/ee/spec/features/search/elastic/global_search_spec.rb index 5a484f58b74fa6..22c2adcf1a6dcc 100644 --- a/ee/spec/features/search/elastic/global_search_spec.rb +++ b/ee/spec/features/search/elastic/global_search_spec.rb @@ -277,10 +277,6 @@ end context 'when block_anonymous_global_searches is enabled' do - before do - stub_feature_flags(block_anonymous_global_searches: true) - end - it_behaves_like 'a redacted search results page', include_anonymous: false do let(:search_path) { explore_root_path } end -- GitLab From ed792b5b0c890a6ea1a2a3d8c039d45f60cda30f Mon Sep 17 00:00:00 2001 From: Terri Chu Date: Wed, 9 Sep 2020 20:46:28 -0400 Subject: [PATCH 07/10] fixup! Revert "Revert "Merge branch..."" --- ee/spec/services/search/global_service_spec.rb | 4 ---- spec/controllers/search_controller_spec.rb | 4 ---- spec/features/search/user_searches_for_issues_spec.rb | 1 - spec/features/search/user_searches_for_projects_spec.rb | 4 ---- 4 files changed, 13 deletions(-) diff --git a/ee/spec/services/search/global_service_spec.rb b/ee/spec/services/search/global_service_spec.rb index d7649ca65929ca..d7986ae64bb7dc 100644 --- a/ee/spec/services/search/global_service_spec.rb +++ b/ee/spec/services/search/global_service_spec.rb @@ -198,10 +198,6 @@ end context 'when advanced_global_search_for_limited_indexing feature flag is enabled' do - before do - stub_feature_flags(advanced_global_search_for_limited_indexing: true) - end - it 'includes ES-specific scopes' do expect(described_class.new(user, {}).allowed_scopes).to include('commits') end diff --git a/spec/controllers/search_controller_spec.rb b/spec/controllers/search_controller_spec.rb index 3d883e076a6068..292deea456eb13 100644 --- a/spec/controllers/search_controller_spec.rb +++ b/spec/controllers/search_controller_spec.rb @@ -144,10 +144,6 @@ end context 'when block_anonymous_global_searches is enabled' do - before do - stub_feature_flags(block_anonymous_global_searches: true) - end - context 'for unauthenticated user' do before do sign_out(user) diff --git a/spec/features/search/user_searches_for_issues_spec.rb b/spec/features/search/user_searches_for_issues_spec.rb index e6f05f9d72d463..900ed35adeac7b 100644 --- a/spec/features/search/user_searches_for_issues_spec.rb +++ b/spec/features/search/user_searches_for_issues_spec.rb @@ -108,7 +108,6 @@ def search_for_issue(search) context 'when block_anonymous_global_searches is enabled' do before do - stub_feature_flags(block_anonymous_global_searches: true) visit(search_path) end diff --git a/spec/features/search/user_searches_for_projects_spec.rb b/spec/features/search/user_searches_for_projects_spec.rb index 6116d8c4c874fc..b64909dd42fdd3 100644 --- a/spec/features/search/user_searches_for_projects_spec.rb +++ b/spec/features/search/user_searches_for_projects_spec.rb @@ -40,10 +40,6 @@ end context 'when block_anonymous_global_searches is enabled' do - before do - stub_feature_flags(block_anonymous_global_searches: true) - end - it 'is redirected to login page' do visit(search_path) expect(page).to have_content('You must be logged in to search across all of GitLab') -- GitLab From eac3d5e8fd0d4f7e6ae95fb533fc067bc751b51c Mon Sep 17 00:00:00 2001 From: Terri Chu Date: Wed, 9 Sep 2020 21:19:11 -0400 Subject: [PATCH 08/10] fixup! Revert "Revert "Merge branch..."" --- spec/controllers/search_controller_spec.rb | 76 +++++++++++----------- 1 file changed, 39 insertions(+), 37 deletions(-) diff --git a/spec/controllers/search_controller_spec.rb b/spec/controllers/search_controller_spec.rb index 292deea456eb13..f244392bbad591 100644 --- a/spec/controllers/search_controller_spec.rb +++ b/spec/controllers/search_controller_spec.rb @@ -95,49 +95,51 @@ using RSpec::Parameterized::TableSyntax render_views - before do - stub_feature_flags(block_anonymous_global_searches: false) - end - - it 'omits pipeline status from load' do - project = create(:project, :public) - expect(Gitlab::Cache::Ci::ProjectPipelineStatus).not_to receive(:load_in_batch_for_projects) + context 'when block_anonymous_global_searches is disabled' do + before do + stub_feature_flags(block_anonymous_global_searches: false) + end - get :show, params: { scope: 'projects', search: project.name } + it 'omits pipeline status from load' do + project = create(:project, :public) + expect(Gitlab::Cache::Ci::ProjectPipelineStatus).not_to receive(:load_in_batch_for_projects) - expect(assigns[:search_objects].first).to eq project - end + get :show, params: { scope: 'projects', search: project.name } - context 'check search term length' do - let(:search_queries) do - char_limit = SearchService::SEARCH_CHAR_LIMIT - term_limit = SearchService::SEARCH_TERM_LIMIT - { - chars_under_limit: ('a' * (char_limit - 1)), - chars_over_limit: ('a' * (char_limit + 1)), - terms_under_limit: ('abc ' * (term_limit - 1)), - terms_over_limit: ('abc ' * (term_limit + 1)) - } + expect(assigns[:search_objects].first).to eq project end - where(:string_name, :expectation) do - :chars_under_limit | :not_to_set_flash - :chars_over_limit | :set_chars_flash - :terms_under_limit | :not_to_set_flash - :terms_over_limit | :set_terms_flash - end + context 'check search term length' do + let(:search_queries) do + char_limit = SearchService::SEARCH_CHAR_LIMIT + term_limit = SearchService::SEARCH_TERM_LIMIT + { + chars_under_limit: ('a' * (char_limit - 1)), + chars_over_limit: ('a' * (char_limit + 1)), + terms_under_limit: ('abc ' * (term_limit - 1)), + terms_over_limit: ('abc ' * (term_limit + 1)) + } + end + + where(:string_name, :expectation) do + :chars_under_limit | :not_to_set_flash + :chars_over_limit | :set_chars_flash + :terms_under_limit | :not_to_set_flash + :terms_over_limit | :set_terms_flash + end - with_them do - it do - get :show, params: { scope: 'projects', search: search_queries[string_name] } - - case expectation - when :not_to_set_flash - expect(controller).not_to set_flash[:alert] - when :set_chars_flash - expect(controller).to set_flash[:alert].to(/characters/) - when :set_terms_flash - expect(controller).to set_flash[:alert].to(/terms/) + with_them do + it do + get :show, params: { scope: 'projects', search: search_queries[string_name] } + + case expectation + when :not_to_set_flash + expect(controller).not_to set_flash[:alert] + when :set_chars_flash + expect(controller).to set_flash[:alert].to(/characters/) + when :set_terms_flash + expect(controller).to set_flash[:alert].to(/terms/) + end end end end -- GitLab From d36e63aba5ee3673c549a21d772db9bd636da25b Mon Sep 17 00:00:00 2001 From: Terri Chu Date: Wed, 9 Sep 2020 21:22:04 -0400 Subject: [PATCH 09/10] fixup! Revert "Revert "Merge branch..."" --- ee/spec/services/search/global_service_spec.rb | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/ee/spec/services/search/global_service_spec.rb b/ee/spec/services/search/global_service_spec.rb index d7986ae64bb7dc..88a281fe6a70ae 100644 --- a/ee/spec/services/search/global_service_spec.rb +++ b/ee/spec/services/search/global_service_spec.rb @@ -190,11 +190,16 @@ context 'when elasticsearch_limit_indexing is enabled' do before do stub_ee_application_setting(elasticsearch_limit_indexing: true) - stub_feature_flags(advanced_global_search_for_limited_indexing: false) end - it 'does not include ES-specific scopes' do - expect(described_class.new(user, {}).allowed_scopes).not_to include('commits') + context 'when advanced_global_search_for_limited_indexing feature flag is disabled' do + before do + stub_feature_flags(advanced_global_search_for_limited_indexing: false) + end + + it 'does not include ES-specific scopes' do + expect(described_class.new(user, {}).allowed_scopes).not_to include('commits') + end end context 'when advanced_global_search_for_limited_indexing feature flag is enabled' do -- GitLab From 70394388c9440ddffca3dae703f93540e676b1c0 Mon Sep 17 00:00:00 2001 From: Terri Chu Date: Thu, 10 Sep 2020 09:10:04 -0400 Subject: [PATCH 10/10] fixup! Revert "Revert "Merge branch..."" --- app/controllers/search_controller.rb | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/app/controllers/search_controller.rb b/app/controllers/search_controller.rb index 5b623a4fb93164..d3260e3b60926a 100644 --- a/app/controllers/search_controller.rb +++ b/app/controllers/search_controller.rb @@ -131,11 +131,13 @@ def append_info_to_payload(payload) end def block_anonymous_global_searches - if params[:project_id].blank? && params[:group_id].blank? && !current_user && ::Feature.enabled?(:block_anonymous_global_searches) - store_location_for(:user, request.fullpath) + return if params[:project_id].present? || params[:group_id].present? + return if current_user + return unless ::Feature.enabled?(:block_anonymous_global_searches) - redirect_to new_user_session_path, alert: _('You must be logged in to search across all of GitLab') - end + store_location_for(:user, request.fullpath) + + redirect_to new_user_session_path, alert: _('You must be logged in to search across all of GitLab') end end -- GitLab