From 40ce2084608e91342f00159adf3b75e5abc2efa8 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Thu, 11 Dec 2025 23:57:12 +0100 Subject: [PATCH 01/11] Add elasticsearch_code_scope in Admin setting and elastic:info rake task Admins can now disable code search with elastic. Changelog: added MR: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/215594 EE: true --- doc/integration/advanced_search/elasticsearch.md | 1 + ee/app/helpers/ee/application_settings_helper.rb | 1 + .../services/search/rake_task_executor_service.rb | 15 ++++++--------- .../_elasticsearch_form.html.haml | 3 +++ .../ee/application_settings_helper_spec.rb | 1 + .../search/rake_task_executor_service_spec.rb | 9 ++++++++- locale/gitlab.pot | 6 ++++++ 7 files changed, 26 insertions(+), 10 deletions(-) diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index b8df5ee3027c89..635b4e1b93b8b0 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -562,6 +562,7 @@ The following Elasticsearch settings are available: | **Turn on indexing for advanced search** | Turns on or turns off indexing and creates an empty index if one does not already exist. You may want to turn on indexing but turn off search to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. | | **Pause indexing for advanced search** | Pauses advanced search indexing. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the index until resumed. | | **Search with advanced search** | Turns on or turns off the advanced search capabilities in search and [advanced vulnerability management](../../user/application_security/vulnerability_report/_index.md#advanced-vulnerability-management). | +| **Code search with advanced search** | Turns on or turns off using advanced search in code search. If you uncheck this option, all code data will be deleted from the Elasticsearch. If you want to enable it again, you need to perform full reindexing for code. If you have Zoekt enabled, then you can consider to uncheck this option to save some resources. | | **Requeue indexing workers** | Turns on automatic requeuing of indexing workers. This improves non-code indexing throughput by enqueuing Sidekiq jobs until all documents are processed. Requeuing indexing workers is not recommended for smaller instances or instances with few Sidekiq processes. | | **URL** | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields. Alternatively, use inline credentials such as `http://:@:9200/`. If you use [OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/vpc.html), only connections over ports `80` and `443` are accepted. | | **Username** | The `username` of your Elasticsearch instance. | diff --git a/ee/app/helpers/ee/application_settings_helper.rb b/ee/app/helpers/ee/application_settings_helper.rb index c7d1d261db535d..dc21b2245a2b6c 100644 --- a/ee/app/helpers/ee/application_settings_helper.rb +++ b/ee/app/helpers/ee/application_settings_helper.rb @@ -51,6 +51,7 @@ def visible_attributes :elasticsearch_analyzers_smartcn_search, :elasticsearch_analyzers_kuromoji_enabled, :elasticsearch_analyzers_kuromoji_search, + :elasticsearch_code_scope, :enforce_namespace_storage_limit, :geo_node_allowed_ips, :geo_status_timeout, diff --git a/ee/app/services/search/rake_task_executor_service.rb b/ee/app/services/search/rake_task_executor_service.rb index ca1fbecdd65b4f..ccd8f19d6cff4b 100644 --- a/ee/app/services/search/rake_task_executor_service.rb +++ b/ee/app/services/search/rake_task_executor_service.rb @@ -529,20 +529,17 @@ def display_search_application_settings logger.info("Indexing enabled:\t\t#{setting.elasticsearch_indexing? ? Rainbow('yes').green : 'no'}") logger.info("Search enabled:\t\t\t#{setting.elasticsearch_search? ? Rainbow('yes').green : 'no'}") - logger.info("Requeue Indexing workers:\t" \ - "#{setting.elasticsearch_requeue_workers? ? Rainbow('yes').green : 'no'}") - logger.info("Pause indexing:\t\t\t" \ - "#{setting.elasticsearch_pause_indexing? ? Rainbow('yes').green : 'no'}") + logger.info("Code Search enabled:\t\t#{setting.elasticsearch_code_scope? ? Rainbow('yes').green : 'no'}") + logger.info("Requeue Indexing workers:\t#{setting.elasticsearch_requeue_workers? ? Rainbow('yes').green : 'no'}") + logger.info("Pause indexing:\t\t\t#{setting.elasticsearch_pause_indexing? ? Rainbow('yes').green : 'no'}") logger.info("Indexing restrictions enabled:\t" \ "#{setting.elasticsearch_limit_indexing? ? Rainbow('yes').yellow : 'no'}") logger.info("File size limit:\t\t#{setting.elasticsearch_indexed_file_size_limit_kb} KiB") logger.info("Index version:\t\t\t#{current_index_version}") - logger.info("Indexing number of shards:\t" \ - "#{::Elastic::ProcessBookkeepingService.active_number_of_shards}") - logger.info("Max code indexing concurrency:\t" \ - "#{setting.elasticsearch_max_code_indexing_concurrency}") + logger.info("Indexing number of shards:\t#{::Elastic::ProcessBookkeepingService.active_number_of_shards}") + logger.info("Max code indexing concurrency:\t#{setting.elasticsearch_max_code_indexing_concurrency}") logger.info("Prefix:\t\t\t\t#{setting.elasticsearch_prefix}") - logger.info("Client adapter:\t\t\t\t#{setting.elasticsearch_client_adapter}") + logger.info("Client adapter:\t\t\t#{setting.elasticsearch_client_adapter}") end def display_search_server_info diff --git a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml index f1e22176a34d40..2f751df076213d 100644 --- a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml +++ b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml @@ -48,6 +48,9 @@ .form-group = f.gitlab_ui_checkbox_component :elasticsearch_search, s_('AdminSettings|Search with advanced search'), checkbox_options: { data: { testid: 'search-checkbox' } }, help_text: s_('AdminSettings|Turn off advanced search until indexing is complete.') + .form-group + = f.gitlab_ui_checkbox_component :elasticsearch_code_scope, s_('AdminSettings|Code search with advanced search'), checkbox_options: { data: { testid: 'code-search-checkbox' } }, help_text: s_('AdminSettings|Unchecking this option disables code search using Elasticsearch. It is recommended, to uncheck this if you have zoekt search enabled.') + .form-group = f.gitlab_ui_checkbox_component :elasticsearch_requeue_workers, s_('AdminSettings|Requeue indexing workers'), help_text: s_('AdminSettings|Improve non-code indexing throughput by enqueuing Sidekiq jobs until all documents are processed.') diff --git a/ee/spec/helpers/ee/application_settings_helper_spec.rb b/ee/spec/helpers/ee/application_settings_helper_spec.rb index 56b0dd8616cefd..0fd1aac3fe67b3 100644 --- a/ee/spec/helpers/ee/application_settings_helper_spec.rb +++ b/ee/spec/helpers/ee/application_settings_helper_spec.rb @@ -17,6 +17,7 @@ it 'contains search parameters' do expected_fields = %i[ + elasticsearch_code_scope global_search_code_enabled global_search_commits_enabled global_search_wiki_enabled diff --git a/ee/spec/services/search/rake_task_executor_service_spec.rb b/ee/spec/services/search/rake_task_executor_service_spec.rb index 940daabec31f84..abaa65d529dfde 100644 --- a/ee/spec/services/search/rake_task_executor_service_spec.rb +++ b/ee/spec/services/search/rake_task_executor_service_spec.rb @@ -1044,9 +1044,16 @@ expected_regex = [ /Indexing enabled:\s+yes/, /Search enabled:\s+yes/, + /Code Search enabled:\s+yes/, /Requeue Indexing workers:\s+no/, /Pause indexing:\s+no/, - /Indexing restrictions enabled:\s+no/ + /Indexing restrictions enabled:\s+no/, + /File size limit:\s+#{settings.elasticsearch_indexed_file_size_limit_kb} KiB/, + /Index version:\s+\d+/, + /Indexing number of shards:\s+\d+/, + /Max code indexing concurrency:\s+#{settings.elasticsearch_max_code_indexing_concurrency}/, + /Prefix:\s+#{settings.elasticsearch_prefix}/, + /Client adapter:\s+#{settings.elasticsearch_client_adapter}/ ] expected_regex.each do |expected| diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 04967cd8ef03ee..145e02dc3108ff 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -5640,6 +5640,9 @@ msgstr "" msgid "AdminSettings|Code can be imported from enabled sources during project creation. OmniAuth must be configured for GitHub %{github_docs_link_start}%{icon}%{github_docs_link_end} and Bitbucket %{bitbucket_docs_link_start}%{icon}%{bitbucket_docs_link_end}." msgstr "" +msgid "AdminSettings|Code search with advanced search" +msgstr "" + msgid "AdminSettings|Collector host" msgstr "" @@ -6093,6 +6096,9 @@ msgstr "" msgid "AdminSettings|Turn on indexing for advanced search" msgstr "" +msgid "AdminSettings|Unchecking this option disables code search using Elasticsearch. It is recommended, to uncheck this if you have zoekt search enabled." +msgstr "" + msgid "AdminSettings|Use AWS OpenSearch Service with IAM credentials" msgstr "" -- GitLab From f74079a0e777163cd952c06b756bdb7145de63c1 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Fri, 12 Dec 2025 01:03:07 +0100 Subject: [PATCH 02/11] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: GitLab Duo --- .../admin/application_settings/_elasticsearch_form.html.haml | 2 +- locale/gitlab.pot | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml index 2f751df076213d..27b85e278b02dd 100644 --- a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml +++ b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml @@ -49,7 +49,7 @@ = f.gitlab_ui_checkbox_component :elasticsearch_search, s_('AdminSettings|Search with advanced search'), checkbox_options: { data: { testid: 'search-checkbox' } }, help_text: s_('AdminSettings|Turn off advanced search until indexing is complete.') .form-group - = f.gitlab_ui_checkbox_component :elasticsearch_code_scope, s_('AdminSettings|Code search with advanced search'), checkbox_options: { data: { testid: 'code-search-checkbox' } }, help_text: s_('AdminSettings|Unchecking this option disables code search using Elasticsearch. It is recommended, to uncheck this if you have zoekt search enabled.') + = f.gitlab_ui_checkbox_component :elasticsearch_code_scope, s_('AdminSettings|Code search with advanced search'), checkbox_options: { data: { testid: 'code-search-checkbox' } }, help_text: s_('AdminSettings|Unchecking this option disables code search using Elasticsearch. It is recommended to uncheck this if you have Zoekt search enabled.') .form-group = f.gitlab_ui_checkbox_component :elasticsearch_requeue_workers, s_('AdminSettings|Requeue indexing workers'), help_text: s_('AdminSettings|Improve non-code indexing throughput by enqueuing Sidekiq jobs until all documents are processed.') diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 145e02dc3108ff..285b3908d94dad 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -6096,7 +6096,7 @@ msgstr "" msgid "AdminSettings|Turn on indexing for advanced search" msgstr "" -msgid "AdminSettings|Unchecking this option disables code search using Elasticsearch. It is recommended, to uncheck this if you have zoekt search enabled." +msgid "AdminSettings|Unchecking this option disables code search using Elasticsearch. It is recommended to uncheck this if you have Zoekt search enabled." msgstr "" msgid "AdminSettings|Use AWS OpenSearch Service with IAM credentials" -- GitLab From 7ae4211319899def4002be72e63a5c218ca160dd Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Fri, 12 Dec 2025 10:52:53 +0100 Subject: [PATCH 03/11] Add logic to delete the blob docs when setting is opted out --- ee/app/models/ee/application_setting.rb | 10 +++ .../elastic/delete/all_blobs_service.rb | 27 +++++++ .../workers/search/elastic/delete_worker.rb | 5 +- ee/spec/models/application_setting_spec.rb | 69 +++++++++++++++++ .../elastic/delete/all_blobs_service_spec.rb | 74 +++++++++++++++++++ .../search/elastic/delete_worker_spec.rb | 23 ++++-- 6 files changed, 199 insertions(+), 9 deletions(-) create mode 100644 ee/app/services/search/elastic/delete/all_blobs_service.rb create mode 100644 ee/spec/services/search/elastic/delete/all_blobs_service_spec.rb diff --git a/ee/app/models/ee/application_setting.rb b/ee/app/models/ee/application_setting.rb index 25b113db46c2f4..16c2fe8513155f 100644 --- a/ee/app/models/ee/application_setting.rb +++ b/ee/app/models/ee/application_setting.rb @@ -394,6 +394,7 @@ module ApplicationSetting after_commit :update_personal_access_tokens_lifetime, if: :saved_change_to_max_personal_access_token_lifetime? after_commit :trigger_clickhouse_for_analytics_enabled_event + after_commit :remove_code_data_from_elasticsearch, if: :elasticsearch_code_scope_opted_out? end class_methods do @@ -947,5 +948,14 @@ def duo_settings_immutable_on_saas errors.add(:base, 'Duo settings cannot be modified on GitLab.com') end + + # Add a small delay to avoid any potential race condition between new indexing and deletion + def remove_code_data_from_elasticsearch + ::Search::Elastic::DeleteWorker.perform_async(task: :delete_all_blobs) + end + + def elasticsearch_code_scope_opted_out? + elasticsearch_code_scope_previously_changed?(from: true, to: false) + end end end diff --git a/ee/app/services/search/elastic/delete/all_blobs_service.rb b/ee/app/services/search/elastic/delete/all_blobs_service.rb new file mode 100644 index 00000000000000..d7820913615040 --- /dev/null +++ b/ee/app/services/search/elastic/delete/all_blobs_service.rb @@ -0,0 +1,27 @@ +# frozen_string_literal: true + +module Search + module Elastic + module Delete + class AllBlobsService < BaseService + private + + def index_name + ::Elastic::Latest::Config.index_name + end + + def build_query + return {} unless Gitlab::CurrentSettings.elasticsearch_code_scope + + { + query: { + term: { + join_field: 'blob' + } + } + } + end + end + end + end +end diff --git a/ee/app/workers/search/elastic/delete_worker.rb b/ee/app/workers/search/elastic/delete_worker.rb index 11d8215a17b05c..acf1428a7ca27c 100644 --- a/ee/app/workers/search/elastic/delete_worker.rb +++ b/ee/app/workers/search/elastic/delete_worker.rb @@ -14,8 +14,9 @@ class DeleteWorker pause_control :advanced_search TASKS = { - delete_project_work_items: ::Search::Elastic::Delete::ProjectWorkItemsService, - delete_project_vulnerabilities: ::Search::Elastic::Delete::VulnerabilityService + delete_all_blobs: ::Search::Elastic::Delete::AllBlobsService, + delete_project_vulnerabilities: ::Search::Elastic::Delete::VulnerabilityService, + delete_project_work_items: ::Search::Elastic::Delete::ProjectWorkItemsService }.freeze def perform(options = {}) diff --git a/ee/spec/models/application_setting_spec.rb b/ee/spec/models/application_setting_spec.rb index 1b54705ca97210..5938d529ce3028 100644 --- a/ee/spec/models/application_setting_spec.rb +++ b/ee/spec/models/application_setting_spec.rb @@ -1507,6 +1507,75 @@ end end + describe 'callbacks' do + describe '#remove_code_data_from_elasticsearch', feature_category: :global_search do + context 'when elasticsearch_code_scope is opted out(true -> false)' do + before do + setting.elasticsearch_code_scope = true + setting.save!(validate: false) + end + + it 'calls Search::Elastic::DeleteWorker' do + expect(Search::Elastic::DeleteWorker).to receive(:perform_async).with(task: :delete_all_blobs) + + setting.update!(elasticsearch_code_scope: false) # opted out + expect(setting.reload.elasticsearch_code_scope).to be false + end + + context 'when some other setting is also changed' do + it 'calls Search::Elastic::DeleteWorker' do + expect(Search::Elastic::DeleteWorker).to receive(:perform_async).with(task: :delete_all_blobs) + + setting.update!(elasticsearch_code_scope: false, elasticsearch_retry_on_failure: 3) + expect(setting.reload.elasticsearch_code_scope).to be false + end + end + end + + context 'when elasticsearch_code_scope is opted in(false -> true)' do + before do + setting.elasticsearch_code_scope = false + setting.save!(validate: false) + end + + it 'does not call Search::Elastic::DeleteWorker' do + expect(Search::Elastic::DeleteWorker).not_to receive(:perform_async) + + setting.update!(elasticsearch_code_scope: true) # opted in + expect(setting.reload.elasticsearch_code_scope).to be true + end + end + + context 'when elasticsearch_code_scope setting is untouched' do + context 'when current elasticsearch_code_scope is true' do + before do + setting.elasticsearch_code_scope = true + setting.save!(validate: false) + end + + it 'does not call Search::Elastic::DeleteWorker' do + expect(Search::Elastic::DeleteWorker).not_to receive(:perform_async) + + setting.update!(elasticsearch_retry_on_failure: 5) + end + end + + context 'when current elasticsearch_code_scope is false' do + before do + setting.elasticsearch_code_scope = false + setting.save!(validate: false) + end + + it 'does not call Search::Elastic::DeleteWorker' do + expect(Search::Elastic::DeleteWorker).not_to receive(:perform_async) + + setting.update!(elasticsearch_retry_on_failure: 5) + end + end + end + end + end + describe 'search curation settings after .create_from_defaults', feature_category: :global_search do it { expect(setting.search_max_shard_size_gb).to eq(1) } it { expect(setting.search_max_docs_denominator).to eq(100) } diff --git a/ee/spec/services/search/elastic/delete/all_blobs_service_spec.rb b/ee/spec/services/search/elastic/delete/all_blobs_service_spec.rb new file mode 100644 index 00000000000000..02fff0460c581b --- /dev/null +++ b/ee/spec/services/search/elastic/delete/all_blobs_service_spec.rb @@ -0,0 +1,74 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Search::Elastic::Delete::AllBlobsService, feature_category: :global_search do + let(:main_index) { Elastic::Latest::Config.index_name } + + describe 'integration', :elastic_delete_by_query, :elasticsearch_settings_enabled do + context 'when blobs are present in index', :sidekiq_inline do + let_it_be(:project) { create(:project, :small_repo) } + + before do + project.repository.index_commits_and_blobs + create(:personal_snippet) + ensure_elasticsearch_index! + end + + it 'only deletes all blob documents from the main index' do + # Verify index has documents + initial_blob_docs, initial_non_blob_docs = docs_in_index_partition_by_type_blobs + expect(initial_blob_docs).not_to be_empty + expect(initial_non_blob_docs).not_to be_empty + + described_class.execute({}) + + # Refresh the index to make deletions visible + es_helper.refresh_index(index_name: main_index) + + # Verify only blob documents are deleted + final_blob_docs, final_non_blob_docs = docs_in_index_partition_by_type_blobs + expect(final_blob_docs).to be_empty + expect(final_non_blob_docs).not_to be_empty + end + + context 'when setting elasticsearch_code_scope is disabled' do + before do + stub_ee_application_setting(elasticsearch_code_scope: false) + end + + it 'does not delete any document' do + # Verify index has documents + initial_blob_docs, initial_non_blob_docs = docs_in_index_partition_by_type_blobs + expect(initial_blob_docs).not_to be_empty + expect(initial_non_blob_docs).not_to be_empty + + described_class.execute({}) + + # Refresh the index to make deletions visible + es_helper.refresh_index(index_name: main_index) + + # Verify index still has documents + initial_blob_docs, initial_non_blob_docs = docs_in_index_partition_by_type_blobs + expect(initial_blob_docs).not_to be_empty + expect(initial_non_blob_docs).not_to be_empty + end + end + end + + context 'when no blobs are present in index' do + it 'completes successfully without errors' do + # Verify index has no documents + initial_blobs_docs, initial_non_blobs_docs = docs_in_index_partition_by_type_blobs + expect(initial_blobs_docs).to be_empty + expect(initial_non_blobs_docs).to be_empty + + expect { described_class.execute({}) }.not_to raise_error + end + end + + def docs_in_index_partition_by_type_blobs + items_in_index(main_index, source: true).partition { |doc| doc.dig('join_field', 'name') == 'blob' } + end + end +end diff --git a/ee/spec/workers/search/elastic/delete_worker_spec.rb b/ee/spec/workers/search/elastic/delete_worker_spec.rb index 7a4602fd6d525a..d5b48ddb31da65 100644 --- a/ee/spec/workers/search/elastic/delete_worker_spec.rb +++ b/ee/spec/workers/search/elastic/delete_worker_spec.rb @@ -39,10 +39,10 @@ end context 'when we pass valid task' do + subject(:perform) { described_class.new.perform({ task: task }) } + context 'with delete_project_work_items task' do - subject(:perform) do - described_class.new.perform({ task: :delete_project_work_items }) - end + let(:task) { :delete_project_work_items } it 'calls the corresponding service' do expect(::Search::Elastic::Delete::ProjectWorkItemsService).to receive(:execute) @@ -51,20 +51,29 @@ end context 'with delete_project_vulnerabilities task' do - subject(:perform) do - described_class.new.perform({ task: :delete_project_vulnerabilities }) - end + let(:task) { :delete_project_vulnerabilities } it 'calls the corresponding service' do expect(::Search::Elastic::Delete::VulnerabilityService).to receive(:execute) perform end end + + context 'with delete_all_blobs task' do + let(:task) { :delete_all_blobs } + + it 'calls the corresponding service' do + expect(::Search::Elastic::Delete::AllBlobsService).to receive(:execute) + perform + end + end end context 'when we pass invalid task' do + let(:task) { :unknown_task } + it 'raises ArgumentError' do - expect { described_class.new.perform({ task: :unknown_task }) }.to raise_error(ArgumentError) + expect { perform }.to raise_error(ArgumentError) end end end -- GitLab From fdfcf1c3e23b0e582c5fcf510d564b68aeccc92a Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Mon, 15 Dec 2025 13:20:40 +0100 Subject: [PATCH 04/11] Apply 2 suggestion(s) to 2 file(s) Co-authored-by: Ashraf Khamis --- doc/integration/advanced_search/elasticsearch.md | 2 +- .../application_settings/_elasticsearch_form.html.haml | 2 +- locale/gitlab.pot | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index 635b4e1b93b8b0..c74f68e870d21f 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -562,7 +562,7 @@ The following Elasticsearch settings are available: | **Turn on indexing for advanced search** | Turns on or turns off indexing and creates an empty index if one does not already exist. You may want to turn on indexing but turn off search to give the index time to be fully completed, for example. Also, keep in mind that this option doesn't have any impact on existing data, this only enables/disables the background indexer which tracks data changes and ensures new data is indexed. | | **Pause indexing for advanced search** | Pauses advanced search indexing. This is useful for cluster migration/reindexing. All changes are still tracked, but they are not committed to the index until resumed. | | **Search with advanced search** | Turns on or turns off the advanced search capabilities in search and [advanced vulnerability management](../../user/application_security/vulnerability_report/_index.md#advanced-vulnerability-management). | -| **Code search with advanced search** | Turns on or turns off using advanced search in code search. If you uncheck this option, all code data will be deleted from the Elasticsearch. If you want to enable it again, you need to perform full reindexing for code. If you have Zoekt enabled, then you can consider to uncheck this option to save some resources. | +| **Code search with advanced search** | Turns on or turns off code search with advanced search. When this setting is turned off, all code is deleted from your Elasticsearch instance. To turn this setting back on, fully reindex your code. If exact code search is enabled, you should turn off this setting to save resources. | | **Requeue indexing workers** | Turns on automatic requeuing of indexing workers. This improves non-code indexing throughput by enqueuing Sidekiq jobs until all documents are processed. Requeuing indexing workers is not recommended for smaller instances or instances with few Sidekiq processes. | | **URL** | The URL of your Elasticsearch instance. Use a comma-separated list to support clustering (for example, `http://host1, https://host2:9200`). If your Elasticsearch instance is password-protected, use the `Username` and `Password` fields. Alternatively, use inline credentials such as `http://:@:9200/`. If you use [OpenSearch](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/vpc.html), only connections over ports `80` and `443` are accepted. | | **Username** | The `username` of your Elasticsearch instance. | diff --git a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml index 27b85e278b02dd..f2288921c8ebaf 100644 --- a/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml +++ b/ee/app/views/admin/application_settings/_elasticsearch_form.html.haml @@ -49,7 +49,7 @@ = f.gitlab_ui_checkbox_component :elasticsearch_search, s_('AdminSettings|Search with advanced search'), checkbox_options: { data: { testid: 'search-checkbox' } }, help_text: s_('AdminSettings|Turn off advanced search until indexing is complete.') .form-group - = f.gitlab_ui_checkbox_component :elasticsearch_code_scope, s_('AdminSettings|Code search with advanced search'), checkbox_options: { data: { testid: 'code-search-checkbox' } }, help_text: s_('AdminSettings|Unchecking this option disables code search using Elasticsearch. It is recommended to uncheck this if you have Zoekt search enabled.') + = f.gitlab_ui_checkbox_component :elasticsearch_code_scope, s_('AdminSettings|Code search with advanced search'), checkbox_options: { data: { testid: 'code-search-checkbox' } }, help_text: s_('AdminSettings|If exact code search is enabled, you should turn off this setting to save resources.') .form-group = f.gitlab_ui_checkbox_component :elasticsearch_requeue_workers, s_('AdminSettings|Requeue indexing workers'), help_text: s_('AdminSettings|Improve non-code indexing throughput by enqueuing Sidekiq jobs until all documents are processed.') diff --git a/locale/gitlab.pot b/locale/gitlab.pot index 285b3908d94dad..2615069c46dc4b 100644 --- a/locale/gitlab.pot +++ b/locale/gitlab.pot @@ -5790,6 +5790,9 @@ msgstr "" msgid "AdminSettings|If GitLab manages your cluster, then GitLab retains your analytics data for 1 year. %{link_start}Learn more about data retention policy%{link_end}." msgstr "" +msgid "AdminSettings|If exact code search is enabled, you should turn off this setting to save resources." +msgstr "" + msgid "AdminSettings|If no unit is written, it defaults to seconds. For example, these are all equivalent: %{oneDayInSeconds}, %{oneDayInHoursHumanReadable}, or %{oneDayHumanReadable}. Minimum value is two hours. %{linkStart}Learn more.%{linkEnd}" msgstr "" @@ -6096,9 +6099,6 @@ msgstr "" msgid "AdminSettings|Turn on indexing for advanced search" msgstr "" -msgid "AdminSettings|Unchecking this option disables code search using Elasticsearch. It is recommended to uncheck this if you have Zoekt search enabled." -msgstr "" - msgid "AdminSettings|Use AWS OpenSearch Service with IAM credentials" msgstr "" -- GitLab From b738e5e04d85cc2dce6b9a47c5a23fe26cde5735 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Mon, 15 Dec 2025 13:56:26 +0100 Subject: [PATCH 05/11] --no-edit --- doc/integration/advanced_search/elasticsearch.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index c74f68e870d21f..236099ac9324cd 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -553,6 +553,19 @@ To enable search with advanced search in GitLab: 1. Select the **Search with advanced search** checkbox. 1. Select **Save changes**. +### Disable code search with advanced search + +Prerequisites: + +- You must have administrator access to the instance. + +To disable search with advanced search in GitLab: + +1. In the upper-right corner, select **Admin**. +1. Select **Settings** > **Search**. +1. Uncheck the **Code search with advanced search** checkbox. +1. Select **Save changes**. + ### Advanced search configuration The following Elasticsearch settings are available: -- GitLab From 8158fc5d15e91612d5c4b905163921f5c331fbc9 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Mon, 15 Dec 2025 14:54:43 +0100 Subject: [PATCH 06/11] Apply 1 suggestion(s) to 1 file(s) --- ee/app/models/ee/application_setting.rb | 1 - 1 file changed, 1 deletion(-) diff --git a/ee/app/models/ee/application_setting.rb b/ee/app/models/ee/application_setting.rb index 16c2fe8513155f..4c89e20f3e53d6 100644 --- a/ee/app/models/ee/application_setting.rb +++ b/ee/app/models/ee/application_setting.rb @@ -949,7 +949,6 @@ def duo_settings_immutable_on_saas errors.add(:base, 'Duo settings cannot be modified on GitLab.com') end - # Add a small delay to avoid any potential race condition between new indexing and deletion def remove_code_data_from_elasticsearch ::Search::Elastic::DeleteWorker.perform_async(task: :delete_all_blobs) end -- GitLab From 5cf933b948515ad9a5bd07f06b672e236d8bea7d Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Mon, 15 Dec 2025 15:09:41 +0100 Subject: [PATCH 07/11] Apply 2 suggestion(s) to 1 file(s) Co-authored-by: Ashraf Khamis --- .../advanced_search/elasticsearch.md | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index 236099ac9324cd..56c4d77b301aff 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -553,19 +553,18 @@ To enable search with advanced search in GitLab: 1. Select the **Search with advanced search** checkbox. 1. Select **Save changes**. -### Disable code search with advanced search +### Enable code search with advanced search Prerequisites: - You must have administrator access to the instance. -To disable search with advanced search in GitLab: +To enable code search with advanced search in GitLab: 1. In the upper-right corner, select **Admin**. 1. Select **Settings** > **Search**. -1. Uncheck the **Code search with advanced search** checkbox. +1. Select the **Code search with advanced search** checkbox. 1. Select **Save changes**. - ### Advanced search configuration The following Elasticsearch settings are available: @@ -736,6 +735,19 @@ To disable search with advanced search in GitLab: 1. Clear the **Search with advanced search** checkbox. 1. Select **Save changes**. +### Disable code search with advanced search + +Prerequisites: + +- You must have administrator access to the instance. + +To disable code search with advanced search in GitLab: + +1. In the upper-right corner, select **Admin**. +1. Select **Settings** > **Search**. +1. Clear the **Code search with advanced search** checkbox. +1. Select **Save changes**. + ## Pause indexing Prerequisites: -- GitLab From 9a83b9edaa4f9f33f45599e91bedb1a363d80013 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Mon, 15 Dec 2025 15:10:31 +0100 Subject: [PATCH 08/11] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Ashraf Khamis --- doc/integration/advanced_search/elasticsearch.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/integration/advanced_search/elasticsearch.md b/doc/integration/advanced_search/elasticsearch.md index 56c4d77b301aff..e4cfcc542e347b 100644 --- a/doc/integration/advanced_search/elasticsearch.md +++ b/doc/integration/advanced_search/elasticsearch.md @@ -565,6 +565,7 @@ To enable code search with advanced search in GitLab: 1. Select **Settings** > **Search**. 1. Select the **Code search with advanced search** checkbox. 1. Select **Save changes**. + ### Advanced search configuration The following Elasticsearch settings are available: -- GitLab From 6cc4f39b31134e46b323c4adaaaa9f1889051040 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Tue, 16 Dec 2025 12:42:47 +0100 Subject: [PATCH 09/11] Fix the undercoverage --- .../search/rake_task_executor_service.rb | 5 +- .../search/rake_task_executor_service_spec.rb | 103 +++++++++++++++++- 2 files changed, 99 insertions(+), 9 deletions(-) diff --git a/ee/app/services/search/rake_task_executor_service.rb b/ee/app/services/search/rake_task_executor_service.rb index ccd8f19d6cff4b..832b63525167e0 100644 --- a/ee/app/services/search/rake_task_executor_service.rb +++ b/ee/app/services/search/rake_task_executor_service.rb @@ -524,12 +524,11 @@ def check_handler end def display_search_application_settings - setting = ::ApplicationSetting.current + setting = Gitlab::CurrentSettings current_index_version = helper.get_meta&.dig('created_by') - logger.info("Indexing enabled:\t\t#{setting.elasticsearch_indexing? ? Rainbow('yes').green : 'no'}") logger.info("Search enabled:\t\t\t#{setting.elasticsearch_search? ? Rainbow('yes').green : 'no'}") - logger.info("Code Search enabled:\t\t#{setting.elasticsearch_code_scope? ? Rainbow('yes').green : 'no'}") + logger.info("Code search enabled:\t\t#{setting.elasticsearch_code_scope? ? Rainbow('yes').green : 'no'}") logger.info("Requeue Indexing workers:\t#{setting.elasticsearch_requeue_workers? ? Rainbow('yes').green : 'no'}") logger.info("Pause indexing:\t\t\t#{setting.elasticsearch_pause_indexing? ? Rainbow('yes').green : 'no'}") logger.info("Indexing restrictions enabled:\t" \ diff --git a/ee/spec/services/search/rake_task_executor_service_spec.rb b/ee/spec/services/search/rake_task_executor_service_spec.rb index abaa65d529dfde..0b5f73f0d1f22c 100644 --- a/ee/spec/services/search/rake_task_executor_service_spec.rb +++ b/ee/spec/services/search/rake_task_executor_service_spec.rb @@ -1040,14 +1040,105 @@ info end + context 'for yes/no settings' do + it 'outputs Indexing enabled as yes when indexing is enabled' do + allow(settings).to receive(:elasticsearch_indexing?).and_return(true) + + expect(logger).to receive(:info).with(/Indexing enabled:\s+yes/) + + info + end + + it 'outputs Indexing enabled as no when indexing is disabled' do + expect(logger).to receive(:info).with(/Indexing enabled:\s+no/) + allow(settings).to receive(:elasticsearch_indexing?).and_return(false) + + info + end + + it 'outputs Search enabled as yes when search is enabled' do + allow(settings).to receive(:elasticsearch_search?).and_return(true) + + expect(logger).to receive(:info).with(/Search enabled:\s+yes/) + + info + end + + it 'outputs Search enabled as no when search is disabled' do + allow(settings).to receive(:elasticsearch_search?).and_return(false) + + expect(logger).to receive(:info).with(/Search enabled:\s+no/) + + info + end + + it 'outputs Code search enabled as yes when code search is enabled' do + allow(settings).to receive(:elasticsearch_code_scope?).and_return(true) + + expect(logger).to receive(:info).with(/Code search enabled:\s+yes/) + + info + end + + it 'outputs Code search enabled as no when code search is disabled' do + allow(settings).to receive(:elasticsearch_code_scope?).and_return(false) + + expect(logger).to receive(:info).with(/Code search enabled:\s+no/) + + info + end + + it 'outputs Requeue Indexing workers as yes when requeue workers is enabled' do + allow(settings).to receive(:elasticsearch_requeue_workers?).and_return(true) + + expect(logger).to receive(:info).with(/Requeue Indexing workers:\s+yes/) + + info + end + + it 'outputs Requeue Indexing workers as no when requeue workers is disabled' do + allow(settings).to receive(:elasticsearch_requeue_workers?).and_return(false) + + expect(logger).to receive(:info).with(/Requeue Indexing workers:\s+no/) + + info + end + + it 'outputs Pause indexing as yes when pause indexing is enabled' do + allow(settings).to receive(:elasticsearch_pause_indexing?).and_return(true) + + expect(logger).to receive(:info).with(/Pause indexing:\s+yes/) + + info + end + + it 'outputs Pause indexing as no when pause indexing is disabled' do + allow(settings).to receive(:elasticsearch_pause_indexing?).and_return(false) + + expect(logger).to receive(:info).with(/Pause indexing:\s+no/) + + info + end + + it 'outputs Indexing restrictions enabled as yes when limit indexing is enabled' do + allow(settings).to receive(:elasticsearch_limit_indexing?).and_return(true) + + expect(logger).to receive(:info).with(/Indexing restrictions enabled:\s+yes/) + + info + end + + it 'outputs Indexing restrictions enabled as no when limit indexing is disabled' do + allow(settings).to receive(:elasticsearch_limit_indexing?).and_return(false) + + expect(logger).to receive(:info).with(/Indexing restrictions enabled:\s+no/) + + info + end + end + it 'outputs indexing and search settings' do expected_regex = [ - /Indexing enabled:\s+yes/, - /Search enabled:\s+yes/, - /Code Search enabled:\s+yes/, - /Requeue Indexing workers:\s+no/, - /Pause indexing:\s+no/, - /Indexing restrictions enabled:\s+no/, /File size limit:\s+#{settings.elasticsearch_indexed_file_size_limit_kb} KiB/, /Index version:\s+\d+/, /Indexing number of shards:\s+\d+/, -- GitLab From 9d3817f8f33b8749092ab4202fc3f3a2a1f6a8b4 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Tue, 16 Dec 2025 15:31:36 +0100 Subject: [PATCH 10/11] Fix the deletion logic --- .../elastic/delete/all_blobs_service.rb | 4 +-- .../elastic/delete/all_blobs_service_spec.rb | 36 +++++++++++-------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/ee/app/services/search/elastic/delete/all_blobs_service.rb b/ee/app/services/search/elastic/delete/all_blobs_service.rb index d7820913615040..b4575a6a69e23f 100644 --- a/ee/app/services/search/elastic/delete/all_blobs_service.rb +++ b/ee/app/services/search/elastic/delete/all_blobs_service.rb @@ -11,12 +11,12 @@ def index_name end def build_query - return {} unless Gitlab::CurrentSettings.elasticsearch_code_scope + return {} if Gitlab::CurrentSettings.elasticsearch_code_scope? { query: { term: { - join_field: 'blob' + type: 'blob' } } } diff --git a/ee/spec/services/search/elastic/delete/all_blobs_service_spec.rb b/ee/spec/services/search/elastic/delete/all_blobs_service_spec.rb index 02fff0460c581b..a300bf45e8a32c 100644 --- a/ee/spec/services/search/elastic/delete/all_blobs_service_spec.rb +++ b/ee/spec/services/search/elastic/delete/all_blobs_service_spec.rb @@ -15,26 +15,32 @@ ensure_elasticsearch_index! end - it 'only deletes all blob documents from the main index' do - # Verify index has documents - initial_blob_docs, initial_non_blob_docs = docs_in_index_partition_by_type_blobs - expect(initial_blob_docs).not_to be_empty - expect(initial_non_blob_docs).not_to be_empty + context 'when setting elasticsearch_code_scope is disabled' do + before do + stub_ee_application_setting(elasticsearch_code_scope: false) + end - described_class.execute({}) + it 'only deletes all blob documents from the main index' do + # Verify index has documents + initial_blob_docs, initial_non_blob_docs = docs_in_index_partition_by_type_blobs + expect(initial_blob_docs).not_to be_empty + expect(initial_non_blob_docs).not_to be_empty - # Refresh the index to make deletions visible - es_helper.refresh_index(index_name: main_index) + described_class.execute({}) + + # Refresh the index to make deletions visible + es_helper.refresh_index(index_name: main_index) - # Verify only blob documents are deleted - final_blob_docs, final_non_blob_docs = docs_in_index_partition_by_type_blobs - expect(final_blob_docs).to be_empty - expect(final_non_blob_docs).not_to be_empty + # Verify only blob documents are deleted + final_blob_docs, final_non_blob_docs = docs_in_index_partition_by_type_blobs + expect(final_blob_docs).to be_empty + expect(final_non_blob_docs).not_to be_empty + end end - context 'when setting elasticsearch_code_scope is disabled' do + context 'when setting elasticsearch_code_scope is enabled' do before do - stub_ee_application_setting(elasticsearch_code_scope: false) + stub_ee_application_setting(elasticsearch_code_scope: true) end it 'does not delete any document' do @@ -68,7 +74,7 @@ end def docs_in_index_partition_by_type_blobs - items_in_index(main_index, source: true).partition { |doc| doc.dig('join_field', 'name') == 'blob' } + items_in_index(main_index, source: true).partition { |doc| doc['type'] == 'blob' } end end end -- GitLab From 268010c1e5776b090bd7ba6f17c0b78cbc220827 Mon Sep 17 00:00:00 2001 From: Ravi Kumar Date: Wed, 17 Dec 2025 11:04:12 +0100 Subject: [PATCH 11/11] Apply 1 suggestion(s) to 1 file(s) Co-authored-by: Dmitry Gruzd --- ee/app/models/ee/application_setting.rb | 2 +- ee/spec/models/application_setting_spec.rb | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ee/app/models/ee/application_setting.rb b/ee/app/models/ee/application_setting.rb index 4c89e20f3e53d6..f5da883d349a23 100644 --- a/ee/app/models/ee/application_setting.rb +++ b/ee/app/models/ee/application_setting.rb @@ -950,7 +950,7 @@ def duo_settings_immutable_on_saas end def remove_code_data_from_elasticsearch - ::Search::Elastic::DeleteWorker.perform_async(task: :delete_all_blobs) + ::Search::Elastic::DeleteWorker.perform_async('task' => 'delete_all_blobs') end def elasticsearch_code_scope_opted_out? diff --git a/ee/spec/models/application_setting_spec.rb b/ee/spec/models/application_setting_spec.rb index 5938d529ce3028..0cf98f4326d4ff 100644 --- a/ee/spec/models/application_setting_spec.rb +++ b/ee/spec/models/application_setting_spec.rb @@ -1516,7 +1516,7 @@ end it 'calls Search::Elastic::DeleteWorker' do - expect(Search::Elastic::DeleteWorker).to receive(:perform_async).with(task: :delete_all_blobs) + expect(Search::Elastic::DeleteWorker).to receive(:perform_async).with('task' => 'delete_all_blobs') setting.update!(elasticsearch_code_scope: false) # opted out expect(setting.reload.elasticsearch_code_scope).to be false @@ -1524,7 +1524,7 @@ context 'when some other setting is also changed' do it 'calls Search::Elastic::DeleteWorker' do - expect(Search::Elastic::DeleteWorker).to receive(:perform_async).with(task: :delete_all_blobs) + expect(Search::Elastic::DeleteWorker).to receive(:perform_async).with('task' => 'delete_all_blobs') setting.update!(elasticsearch_code_scope: false, elasticsearch_retry_on_failure: 3) expect(setting.reload.elasticsearch_code_scope).to be false -- GitLab