503 errors from GraphQL when browsing MRs on a project with a large amount of MRs (>300,000)
Everyone can contribute. Help move this issue forward while earning points, leveling up and collecting rewards.
Summary
503 errors from GraphQL when browsing MRs on a project with a large amount of MRs (>300,000)
Steps to reproduce
Authentic reproduction may be difficult as the sequential creation of MRs on a single project may not reproduce the problem.
At the moment, I can only see this issue on a group with organically created MRs.
Example Project
https://gitlab.com/groups/gitlab-org/-/merge_requests/?sort=closed_at&state=all&first_page_size=20
What is the current bug behavior?
503 error when browsing a group's MRs with a large number of total MRs
What is the expected correct behavior?
The MRs should render as a list
Relevant logs and/or screenshots
JSON error log
{
"correlation_id": "da6b0185c1ff409f2421fdb9fc522869",
"component": "gitlab",
"meta.user": "mbadeau-admin",
"tag": "api-rails.var.log.containers.gitlab-webservice-api-d6cb97687-6rmnb_gitlab_webservice-3b83f9c4372d7deaa0c5f54381d23977a7177b5bc919ae64e6a0c27ec355e400.log",
"type": "api",
"shard": "default",
"meta.organization_id": 1,
"exception.backtrace": [
"lib/gitlab/database/load_balancing/connection_proxy.rb:111:in `public_send'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:111:in `block in read_using_load_balancer'",
"lib/gitlab/database/load_balancing/load_balancer.rb:65:in `read'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:110:in `read_using_load_balancer'",
"lib/gitlab/database/load_balancing/connection_proxy.rb:48:in `select_all'",
"lib/gitlab/graphql/pagination/keyset/connection.rb:122:in `block in limited_nodes'",
"lib/gitlab/graphql/pagination/keyset/connection.rb:108:in `limited_nodes'",
"lib/gitlab/graphql/pagination/keyset/connection.rb:62:in `block in has_next_page'",
"lib/gitlab/graphql/pagination/keyset/connection.rb:58:in `has_next_page'",
"lib/gitlab/graphql/tracers/instrumentation_tracer.rb:23:in `execute_multiplex'",
"app/graphql/gitlab_schema.rb:44:in `multiplex'",
"app/controllers/graphql_controller.rb:253:in `execute_query'",
"app/controllers/graphql_controller.rb:65:in `execute'",
"lib/gitlab/ip_address_state.rb:11:in `with'",
"ee/app/controllers/ee/application_controller.rb:45:in `set_current_ip_address'",
"app/controllers/application_controller.rb:487:in `set_current_admin'",
"lib/gitlab/session.rb:11:in `with_session'",
"app/controllers/application_controller.rb:478:in `set_session_storage'",
"lib/gitlab/i18n.rb:114:in `with_locale'",
"lib/gitlab/i18n.rb:120:in `with_user_locale'",
"app/controllers/application_controller.rb:469:in `set_locale'",
"app/controllers/application_controller.rb:462:in `set_current_context'",
"lib/gitlab/middleware/action_controller_static_context.rb:23:in `call'",
"ee/lib/omni_auth/strategies/group_saml.rb:41:in `other_phase'",
"lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `block in call'",
"lib/gitlab/sidekiq_sharding/validator.rb:42:in `enabled'",
"lib/gitlab/middleware/sidekiq_shard_awareness_validation.rb:20:in `call'",
"lib/gitlab/middleware/memory_report.rb:13:in `call'",
"lib/gitlab/middleware/speedscope.rb:13:in `call'",
"lib/gitlab/database/load_balancing/rack_middleware.rb:23:in `call'",
"lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
"lib/gitlab/etag_caching/middleware.rb:21:in `call'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
"lib/gitlab/metrics/web_transaction.rb:46:in `run'",
"lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
"lib/gitlab/middleware/go.rb:21:in `call'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `block in call'",
"lib/gitlab/database/query_analyzer.rb:83:in `within'",
"lib/gitlab/middleware/query_analyzer.rb:11:in `call'",
"lib/ci/job_token/middleware.rb:11:in `call'",
"lib/gitlab/middleware/multipart.rb:173:in `call'",
"lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
"lib/gitlab/middleware/read_only.rb:18:in `call'",
"lib/gitlab/middleware/unauthenticated_session_expiry.rb:18:in `call'",
"lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
"lib/gitlab/middleware/path_traversal_check.rb:40:in `call'",
"lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
"lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
"lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
"lib/gitlab/middleware/request_context.rb:15:in `call'",
"lib/gitlab/middleware/webhook_recursion_detection.rb:15:in `call'",
"config/initializers/fix_local_cache_middleware.rb:11:in `call'",
"lib/gitlab/middleware/compressed_json.rb:44:in `call'",
"lib/gitlab/middleware/rack_multipart_tempfile_factory.rb:19:in `call'",
"lib/gitlab/middleware/sidekiq_web_static.rb:20:in `call'",
"lib/gitlab/metrics/requests_rack_middleware.rb:83:in `call'",
"lib/gitlab/middleware/release_env.rb:12:in `call'"
],
"query_fingerprint": "getGroupMergeRequestsEE/lGkmNP-7J7cm_S3ZEEXVC8iBkK1rVQ20b87lKjxIKJk=/8/z5QHtKxOqXraaFQ5-vV-ORvAVcZTKvSjpJjTOHN2354=",
"query_analysis.duration_s": 0.004583500000080676,
"environment": "gprd",
"logtag": "F",
"exception.cause_class": "PG::QueryCanceled",
"operation_fingerprint": "getGroupMergeRequestsEE/lGkmNP-7J7cm_S3ZEEXVC8iBkK1rVQ20b87lKjxIKJk=",
"time": "2025-04-02T01:47:39.126Z",
"meta.remote_ip": "123.225.7.131",
"operation_name": "getGroupMergeRequestsEE",
"exception.sql": "/*application:web,correlation_id:da6b0185c1ff409f2421fdb9fc522869,endpoint_id:GraphqlController#execute,db_config_database:gitlabhq_production,db_config_name:main_replica*/ SELECT \"merge_requests\".\"id\", \"merge_requests\".\"target_branch\", \"merge_requests\".\"source_branch\", \"merge_requests\".\"source_project_id\", \"merge_requests\".\"author_id\", \"merge_requests\".\"assignee_id\", \"merge_requests\".\"title\", \"merge_requests\".\"created_at\", \"merge_requests\".\"updated_at\", \"merge_requests\".\"milestone_id\", \"merge_requests\".\"merge_status\", \"merge_requests\".\"target_project_id\", \"merge_requests\".\"iid\", \"merge_requests\".\"description\", \"merge_requests\".\"updated_by_id\", \"merge_requests\".\"merge_error\", \"merge_requests\".\"merge_params\", \"merge_requests\".\"merge_when_pipeline_succeeds\", \"merge_requests\".\"merge_user_id\", \"merge_requests\".\"merge_commit_sha\", \"merge_requests\".\"approvals_before_merge\", \"merge_requests\".\"rebase_commit_sha\", \"merge_requests\".\"in_progress_merge_commit_sha\", \"merge_requests\".\"lock_version\", \"merge_requests\".\"title_html\", \"merge_requests\".\"description_html\", \"merge_requests\".\"time_estimate\", \"merge_requests\".\"squash\", \"merge_requests\".\"cached_markdown_version\", \"merge_requests\".\"last_edited_at\", \"merge_requests\".\"last_edited_by_id\", \"merge_requests\".\"merge_jid\", \"merge_requests\".\"discussion_locked\", \"merge_requests\".\"latest_merge_request_diff_id\", \"merge_requests\".\"allow_maintainer_to_push\", \"merge_requests\".\"state_id\", \"merge_requests\".\"rebase_jid\", \"merge_requests\".\"squash_commit_sha\", \"merge_requests\".\"sprint_id\", \"merge_requests\".\"merge_ref_sha\", \"merge_requests\".\"draft\", \"merge_requests\".\"prepared_at\", \"merge_requests\".\"merged_commit_sha\", \"merge_requests\".\"override_requested_changes\", \"merge_requests\".\"head_pipeline_id\", \"merge_requests\".\"imported_from\", \"merge_requests\".\"retargeted\", \"merge_request_metrics\".\"latest_closed_at\" AS merge_request_metrics_latest_closed_at, \"merge_request_metrics\".\"id\" AS merge_request_metrics_id FROM \"merge_requests\" INNER JOIN \"projects\" ON \"projects\".\"id\" = \"merge_requests\".\"target_project_id\" INNER JOIN \"merge_request_metrics\" ON \"merge_request_metrics\".\"merge_request_id\" = \"merge_requests\".\"id\" LEFT JOIN project_features ON projects.id = project_features.project_id WHERE \"projects\".\"namespace_id\" IN (SELECT \"namespaces\".\"id\" FROM UNNEST(\n COALESCE(\n (SELECT ids FROM (SELECT \"namespace_descendants\".\"self_and_descendant_group_ids\" AS ids FROM \"namespace_descendants\" WHERE \"namespace_descendants\".\"outdated_at\" IS NULL AND \"namespace_descendants\".\"namespace_id\" = $1) cached_query),\n (SELECT ids FROM (SELECT ARRAY_AGG(\"namespaces\".\"id\") AS ids FROM (SELECT namespaces.traversal_ids[array_length(namespaces.traversal_ids, $2)] AS id FROM \"namespaces\" WHERE \"namespaces\".\"type\" = $3 AND (traversal_ids @> ($4))) namespaces) consistent_query))\n) AS namespaces(id)\n) AND (EXISTS (SELECT $5 FROM \"project_authorizations\" WHERE \"project_authorizations\".\"user_id\" = $6 AND (project_authorizations.project_id = projects.id)) OR projects.visibility_level IN ($7,$8,$9)) AND (\"project_features\".\"merge_requests_access_level\" > $10 OR \"project_features\".\"merge_requests_access_level\" IS NULL) AND \"projects\".\"archived\" = $11 AND \"merge_requests\".\"target_project_id\" = \"merge_request_metrics\".\"target_project_id\" ORDER BY \"merge_request_metrics\".\"latest_closed_at\" ASC NULLS LAST, \"merge_request_metrics\".\"id\" DESC LIMIT $12",
"exception.class": "ActiveRecord::QueryCanceled",
"tier": "sv",
"stage": "main",
"exception.message": "PG::QueryCanceled: ERROR: canceling statement due to statement timeout\n",
"meta.client_id": "user/5749275",
"query_analysis.depth": 6,
"meta.http_router_rule_action": "proxy",
"query_analysis.complexity": 130,
"duration_s": 15.050468573999751,
"query_string": "query getGroupMergeRequestsEE($hideUsers: Boolean = false, $isSignedIn: Boolean = false, $fullPath: ID!, $sort: MergeRequestSort, $state: MergeRequestState, $search: String, $approvedBy: [String!], $approver: [String!], $assigneeUsernames: String, $assigneeWildcardId: AssigneeWildcardId, $reviewerUsername: String, $reviewerWildcardId: ReviewerWildcardId, $authorUsername: String, $draft: Boolean, $labelName: [String!], $releaseTag: String, $mergeUser: String, $milestoneTitle: String, $milestoneWildcardId: MilestoneWildcardId, $myReactionEmoji: String, $sourceBranches: [String!], $targetBranches: [String!], $deployedBefore: Time, $deployedAfter: Time, $environmentName: String, $subscribed: SubscriptionStatus, $not: MergeRequestsResolverNegatedParams, $beforeCursor: String, $afterCursor: String, $firstPageSize: Int, $lastPageSize: Int) {\n namespace: group(fullPath: $fullPath) {\n id\n mergeRequests(\n sort: $sort\n state: $state\n search: $search\n approvedBy: $approvedBy\n approver: $approver\n assigneeUsername: $assigneeUsernames\n assigneeWildcardId: $assigneeWildcardId\n reviewerUsername: $reviewerUsername\n reviewerWildcardId: $reviewerWildcardId\n authorUsername: $authorUsername\n draft: $draft\n labelName: $labelName\n releaseTag: $releaseTag\n mergedBy: $mergeUser\n milestoneTitle: $milestoneTitle\n milestoneWildcardId: $milestoneWildcardId\n myReactionEmoji: $myReactionEmoji\n sourceBranches: $sourceBranches\n targetBranches: $targetBranches\n deployedBefore: $deployedBefore\n deployedAfter: $deployedAfter\n environmentName: $environmentName\n subscribed: $subscribed\n not: $not\n includeSubgroups: true\n before: $beforeCursor\n after: $afterCursor\n first: $firstPageSize\n last: $lastPageSize\n ) {\n pageInfo {\n ...PageInfo\n __typename\n }\n nodes {\n ...MergeRequestFragment\n reference(full: true)\n __typename\n }\n __typename\n }\n __typename\n }\n}\n\nfragment PageInfo on PageInfo {\n hasNextPage\n hasPreviousPage\n startCursor\n endCursor\n __typename\n}\n\nfragment MergeRequestFragment on MergeRequest {\n id\n iid\n createdAt\n downvotes\n state\n title\n updatedAt\n mergedAt\n upvotes\n resolvedDiscussionsCount @include(if: $isSignedIn)\n resolvableDiscussionsCount @include(if: $isSignedIn)\n webUrl\n assignees @skip(if: $hideUsers) {\n nodes {\n ...User\n __typename\n }\n __typename\n }\n reviewers @skip(if: $hideUsers) {\n nodes {\n ...User\n __typename\n }\n __typename\n }\n author @skip(if: $hideUsers) {\n ...User\n __typename\n }\n labels {\n nodes {\n id\n color\n title\n description\n __typename\n }\n __typename\n }\n milestone {\n id\n dueDate\n startDate\n webPath\n title\n __typename\n }\n headPipeline {\n id\n detailedStatus {\n id\n icon\n text\n detailsPath\n __typename\n }\n __typename\n }\n conflicts\n commitCount\n sourceBranchExists\n targetBranchExists\n targetBranch\n targetBranchPath\n taskCompletionStatus {\n completedCount\n count\n __typename\n }\n hidden\n __typename\n}\n\nfragment User on User {\n id\n avatarUrl\n name\n username\n webUrl\n webPath\n __typename\n}\n",
"meta.feature_category": "code_review_workflow",
"query_analysis.used_deprecated_fields": [
"DetailedStatus.icon",
"DetailedStatus.text"
],
"is_mutation": false,
"severity": "INFO",
"query_analysis.used_deprecated_arguments": [],
"trace_type": "execute_query",
"subcomponent": "graphql_json",
"meta.caller_id": "graphql:getGroupMergeRequestsEE",
"meta.user_id": 5749275,
"variables": "{\"hideUsers\"=>false, \"isSignedIn\"=>true, \"fullPath\"=>\"gitlab-org\", \"sort\"=>\"CLOSED_AT_ASC\", \"state\"=>\"all\", \"firstPageSize\"=>20, \"afterCursor\"=>nil, \"beforeCursor\"=>nil}",
"query_analysis.used_fields": [
"Group.id",
"PageInfo.hasNextPage",
"PageInfo.hasPreviousPage",
"PageInfo.startCursor",
"PageInfo.endCursor",
"PageInfo.__typename",
"MergeRequestConnection.pageInfo",
"MergeRequest.id",
"MergeRequest.iid",
"MergeRequest.createdAt",
"MergeRequest.downvotes",
"MergeRequest.state",
"MergeRequest.title",
"MergeRequest.updatedAt",
"MergeRequest.mergedAt",
"MergeRequest.upvotes",
"MergeRequest.resolvedDiscussionsCount",
"MergeRequest.resolvableDiscussionsCount",
"MergeRequest.webUrl",
"User.id",
"User.avatarUrl",
"User.name",
"User.username",
"User.webUrl",
"User.webPath",
"User.__typename",
"MergeRequestAssignee.__typename",
"MergeRequestAssigneeConnection.nodes",
"MergeRequestAssigneeConnection.__typename",
"MergeRequest.assignees",
"MergeRequestReviewer.__typename",
"MergeRequestReviewerConnection.nodes",
"MergeRequestReviewerConnection.__typename",
"MergeRequest.reviewers",
"MergeRequestAuthor.__typename",
"MergeRequest.author",
"Label.id",
"Label.color",
"Label.title",
"Label.description",
"Label.__typename",
"LabelConnection.nodes",
"LabelConnection.__typename",
"MergeRequest.labels",
"Milestone.id",
"Milestone.dueDate",
"Milestone.startDate",
"Milestone.webPath",
"Milestone.title",
"Milestone.__typename",
"MergeRequest.milestone",
"Pipeline.id",
"DetailedStatus.id",
"DetailedStatus.icon",
"DetailedStatus.text",
"DetailedStatus.detailsPath",
"DetailedStatus.__typename",
"Pipeline.detailedStatus",
"Pipeline.__typename",
"MergeRequest.headPipeline",
"MergeRequest.conflicts",
"MergeRequest.commitCount",
"MergeRequest.sourceBranchExists",
"MergeRequest.targetBranchExists",
"MergeRequest.targetBranch",
"MergeRequest.targetBranchPath",
"TaskCompletionStatus.completedCount",
"TaskCompletionStatus.count",
"TaskCompletionStatus.__typename",
"MergeRequest.taskCompletionStatus",
"MergeRequest.hidden",
"MergeRequest.__typename",
"MergeRequest.reference",
"MergeRequestConnection.nodes",
"MergeRequestConnection.__typename",
"Group.mergeRequests",
"Group.__typename",
"Query.group"
]
}
}Output of checks
This bug happens on GitLab.com
Results of GitLab environment info
Expand for output related to GitLab environment info
(For installations with omnibus-gitlab package run and paste the output of: `sudo gitlab-rake gitlab:env:info`) (For installations from source run and paste the output of: `sudo -u git -H bundle exec rake gitlab:env:info RAILS_ENV=production`)
Results of GitLab application Check
Expand for output related to the GitLab application check
(For installations with omnibus-gitlab package run and paste the output of:
sudo gitlab-rake gitlab:check SANITIZE=true)(For installations from source run and paste the output of:
sudo -u git -H bundle exec rake gitlab:check RAILS_ENV=production SANITIZE=true)(we will only investigate if the tests are passing)
Possible fixes
Edited by 🤖 GitLab Bot 🤖