[go: up one dir, main page]
Skip to content

Provide a mechanism to disable File Locking

Abstract

Gitlab File Locking is a feature introduced in Gitlab 8.9 https://docs.gitlab.com/ee/user/project/file_lock.html#default-branch-file-and-directory-locks

In a corporate multi-tenant git repository setup, we have historically run into a lot of issue with this feature. Over the year, many users confirmed that they accidentally turn it on while using Gitlab WebUI without having any awareness that the lock existing.

Having the lock existing in the big monorepo also result in a very expensive operation that we encountered recently.

Problem

A few days ago we started receiving complains from our user that some git operations in our monorepo started to fail. Upon digging in further, we encountered issue with Gitaly calling /api/v4/internal/allowed to Gitlab Rail with a 500 error. Stack trace from Gitaly side and log could be found in gitaly#3701 (closed) (I report it as a separate issue).

The root cause of the 500 error on the Rails side was this:

{
  "time": "2021-07-13T18:43:14.828Z",
  "severity": "INFO",
  "duration_s": 60.23882,
  "db_duration_s": 12.71428,
  "view_duration_s": 47.52454,
  "status": 500,
  "method": "POST",
  "path": "/api/v4/internal/allowed",
  "params": [
    {
      "key": "action",
      "value": "git-receive-pack"
    },
    {
      "key": "gl_repository",
      "value": "project-177"
    },
    {
      "key": "project",
      "value": "/var/opt/gitlab/git-data/repositories/@hashed/8c/d2/8cd2510271575d8430c05368315a87b9c4784c7389a47496080c1e615a2a00b6.git"
    },
    {
      "key": "changes",
      "value": "cfc5bd6164bd6f6d2629e4974053b494ef9393c4 aa3c5337ada669740b19760825d20d7ead98e8b4 refs/heads/trunk\n"
    },
    {
      "key": "protocol",
      "value": "ssh"
    },
    {
      "key": "env",
      "value": "{\"GIT_ALTERNATE_OBJECT_DIRECTORIES_RELATIVE\":[\"objects\"],\"GIT_OBJECT_DIRECTORY_RELATIVE\":\"objects/incoming-BKBA8K\"}"
    },
    {
      "key": "user_id",
      "value": "3868"
    }
  ],
  "host": "gitlab.org.com",
  "remote_ip": "10.187.148.19, 127.0.0.1",
  "ua": "gitaly/13.10.3",
  "route": "/api/:version/internal/allowed",
  "exception.class": "Rack::Timeout::RequestTimeoutException",
  "exception.message": "Request ran for longer than 60000ms",
  "exception.backtrace": [
    "ee/lib/gitlab/database/load_balancing/connection_proxy.rb:72:in `block in read_using_load_balancer'",
    "ee/lib/gitlab/database/load_balancing/load_balancer.rb:39:in `read'",
    "ee/lib/gitlab/database/load_balancing/connection_proxy.rb:71:in `read_using_load_balancer'",
    "ee/lib/gitlab/database/load_balancing/connection_proxy.rb:44:in `select_all'",
    "ee/lib/gitlab/path_locks_finder.rb:76:in `find_by_token'",
    "ee/lib/gitlab/path_locks_finder.rb:29:in `block in find'",
    "ee/lib/gitlab/path_locks_finder.rb:28:in `each'",
    "ee/lib/gitlab/path_locks_finder.rb:28:in `find'",
    "ee/app/models/ee/project.rb:547:in `find_path_lock'",
    "ee/lib/ee/gitlab/checks/diff_check.rb:65:in `block in path_locks_validation'",
    "lib/gitlab/checks/diff_check.rb:39:in `block in validate_path'",
    "lib/gitlab/checks/diff_check.rb:38:in `each'",
    "lib/gitlab/checks/diff_check.rb:38:in `validate_path'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in validate_path'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `validate_path'",
    "lib/gitlab/checks/diff_check.rb:19:in `block in validate!'",
    "lib/gitlab/checks/diff_check.rb:18:in `each'",
    "lib/gitlab/checks/diff_check.rb:18:in `validate!'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in validate!'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `validate!'",
    "lib/gitlab/checks/change_access.rb:51:in `commits_check'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in commits_check'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `commits_check'",
    "lib/gitlab/checks/change_access.rb:32:in `validate!'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in validate!'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `validate!'",
    "lib/gitlab/git_access.rb:355:in `check_single_change_access'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in check_single_change_access'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `check_single_change_access'",
    "lib/gitlab/git_access.rb:342:in `block in check_change_access!'",
    "lib/gitlab/git_access.rb:337:in `each'",
    "lib/gitlab/git_access.rb:337:in `with_index'",
    "lib/gitlab/git_access.rb:337:in `check_change_access!'",
    "ee/lib/ee/gitlab/git_access.rb:74:in `check_change_access!'",
    "lib/gitlab/git_access.rb:314:in `check_push_access!'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `block in check_push_access!'",
    "lib/gitlab/metrics/method_call.rb:27:in `measure'",
    "lib/gitlab/metrics/instrumentation.rb:160:in `check_push_access!'",
    "lib/gitlab/git_access.rb:92:in `check'",
    "ee/lib/ee/gitlab/git_access.rb:18:in `check'",
    "lib/api/internal/base.rb:103:in `access_check!'",
    "lib/api/internal/base.rb:56:in `block in check_allowed'",
    "lib/gitlab/auth/current_user_mode.rb:38:in `bypass_session!'",
    "lib/api/internal/base.rb:55:in `check_allowed'",
    "ee/lib/ee/api/internal/base.rb:22:in `block in check_allowed'",
    "ee/lib/gitlab/ip_address_state.rb:10:in `with'",
    "ee/lib/ee/api/internal/base.rb:21:in `check_allowed'",
    "lib/api/internal/base.rb:141:in `block (2 levels) in <class:Base>'",
    "ee/lib/gitlab/middleware/ip_restrictor.rb:11:in `call'",
    "lib/api/api_guard.rb:213:in `call'",
    "lib/gitlab/metrics/elasticsearch_rack_middleware.rb:16:in `call'",
    "lib/gitlab/middleware/rails_queue_duration.rb:33:in `call'",
    "lib/gitlab/metrics/rack_middleware.rb:16:in `block in call'",
    "lib/gitlab/metrics/transaction.rb:56:in `run'",
    "lib/gitlab/metrics/rack_middleware.rb:16:in `call'",
    "lib/gitlab/request_profiler/middleware.rb:17:in `call'",
    "ee/lib/gitlab/database/load_balancing/rack_middleware.rb:39:in `call'",
    "lib/gitlab/jira/middleware.rb:19:in `call'",
    "lib/gitlab/middleware/go.rb:20:in `call'",
    "lib/gitlab/etag_caching/middleware.rb:21:in `call'",
    "lib/gitlab/middleware/multipart.rb:172:in `call'",
    "lib/gitlab/middleware/read_only/controller.rb:50:in `call'",
    "lib/gitlab/middleware/read_only.rb:18:in `call'",
    "lib/gitlab/middleware/same_site_cookies.rb:27:in `call'",
    "lib/gitlab/middleware/handle_malformed_strings.rb:21:in `call'",
    "lib/gitlab/middleware/basic_health_check.rb:25:in `call'",
    "lib/gitlab/middleware/handle_ip_spoof_attack_error.rb:25:in `call'",
    "lib/gitlab/middleware/request_context.rb:21:in `call'",
    "config/initializers/fix_local_cache_middleware.rb:11:in `call'",
    "lib/gitlab/metrics/requests_rack_middleware.rb:76:in `call'",
    "lib/gitlab/middleware/release_env.rb:12:in `call'"
  ],
  "queue_duration_s": 0.005699,
  "gitaly_calls": 5,
  "gitaly_duration_s": 8.015122,
  "redis_calls": 5,
  "redis_duration_s": 0.002177,
  "redis_read_bytes": 541,
  "redis_write_bytes": 233,
  "redis_cache_calls": 4,
  "redis_cache_duration_s": 0.001231,
  "redis_cache_read_bytes": 541,
  "redis_cache_write_bytes": 180,
  "redis_shared_state_calls": 1,
  "redis_shared_state_duration_s": 0.000946,
  "redis_shared_state_write_bytes": 53,
  "db_count": 21878,
  "db_write_count": 29,
  "db_cached_count": 1,
  "db_replica_count": 21878,
  "db_replica_cached_count": 1,
  "db_primary_count": 0,
  "db_primary_cached_count": 0,
  "db_primary_duration_s": 0,
  "db_replica_duration_s": 13.076194,
  "external_http_count": 1,
  "external_http_duration_s": 0.07390435109846294,
  "cpu_s": 41.665822,
  "mem_objects": 7708147,
  "mem_bytes": 156170640,
  "mem_mallocs": 1125481,
  "correlation_id": "01FAGJTFDPHZXEZB8Y2TC2PX8D",
  "meta.user": "user-2",
  "meta.project": "group/repo",
  "meta.root_namespace": "group",
  "meta.caller_id": "/api/:version/internal/allowed",
  "meta.remote_ip": "10.187.148.19",
  "meta.feature_category": "source_code_management",
  "meta.client_id": "user/3868",
  "content_length": "471"
}

As you can see from the stack trace, the existence of a (4 months old) file lock in the repository force Gitaly to trigger expensive query to Postgres to check file lock matching with a large set of files. This ultimately caused a timeout between Rails and Postgres and thus 500 were sent back to Gitaly, failing the git-push.

Upon investigation, we found that we had only 1 lock in the repo which was accidentally enabled by a user without ever disabling it. Disabling the lock help the code path ignore the expensive DB call entirely and git-push to function as normal.

Proposal

Provide a setting that would enable customers to turn off File Locking feature per repository. And perhaps, eventually remove/deprecate this feature from Gitlab.

This File Lock feature caused more confusion to our users. It's not git-native and we also use CODEOWNERS as another form of change approval control.