diff --git a/_support/benchmarking/experiments/master/config.yml b/_support/benchmarking/experiments/master/config.yml index fb58d946b92b037c9e8043823b222e8de0f1c722..1d0fb608c3c885940b4ed183a26fae3673555359 100644 --- a/_support/benchmarking/experiments/master/config.yml +++ b/_support/benchmarking/experiments/master/config.yml @@ -65,6 +65,7 @@ gitaly_instances: # # NOTE: you may wish to delete some of these entries, otherwise benchmarking setup may take a while # to clone each repo. +create_random_small_files: true repositories: - name: git # Whether this repository should be tested. This toggle is read by the K6 script. @@ -73,7 +74,8 @@ repositories: reference_backend: files remote: "https://gitlab.com/gitlab-org/git.git" revision: "2462961280690837670d997bde64bd4ebf8ae66d" - additional_refs: 200000 + additional_refs: 1000 + random_small_file_number: 1000 # Test data to be used as RPC inputs. The K6 script will randomly choose out of these available # inputs. testdata: @@ -94,11 +96,10 @@ repositories: - "Documentation" - "git-gui" - name: git-reftable - include_in_test: true + include_in_test: false reference_backend: reftable remote: "https://gitlab.com/gitlab-org/git.git" revision: "2462961280690837670d997bde64bd4ebf8ae66d" - additional_refs: 200000 testdata: commits: - "fea9d18c534a445ef6e488d8ee711fa92fa0e6bd" @@ -117,11 +118,10 @@ repositories: - "Documentation" - "git-gui" - name: gitlab - include_in_test: true + include_in_test: false reference_backend: files remote: "https://gitlab.com/gitlab-org/gitlab.git" revision: "8f3978675aa4df643cff5a01a8e1896ae754685a" - additional_refs: 1000000 testdata: commits: - "875ffb690e25eb8c98797b5641c6c16c71454b73" @@ -140,11 +140,10 @@ repositories: - "rubocop" - "qa" - name: gitaly - include_in_test: true + include_in_test: false reference_backend: files remote: "https://gitlab.com/gitlab-org/gitaly.git" revision: "4d78df8cb5c6b3abfef5530830dba6c67d9d4c53" - additional_refs: 500000 testdata: commits: - "765d81272feb53bcc0c50199183b4514c5ef7a73" diff --git a/_support/benchmarking/experiments/master/k6-benchmark.js b/_support/benchmarking/experiments/master/k6-benchmark.js index ad21737ea396f8e59e1971b33e1771138ec5a31e..3e9f20db88039fae722f91785c49ae6a708cdb2d 100644 --- a/_support/benchmarking/experiments/master/k6-benchmark.js +++ b/_support/benchmarking/experiments/master/k6-benchmark.js @@ -22,14 +22,14 @@ const optionsStatic = () => { return { scenarios: { - findCommit: { ...SCENARIO_DEFAULTS, rate: 90, exec: 'findCommit' }, - listCommitsByOid: { ...SCENARIO_DEFAULTS, rate: 90, exec: 'listCommitsByOid' }, - getBlobs: { ...SCENARIO_DEFAULTS, rate: 90, exec: 'getBlobs' }, - getTreeEntries: { ...SCENARIO_DEFAULTS, rate: 90, exec: 'getTreeEntries' }, - treeEntry: { ...SCENARIO_DEFAULTS, rate: 40, exec: 'treeEntry' }, - writeAndDeleteRefs: { ...SCENARIO_DEFAULTS, rate: 1, exec: 'writeAndDeleteRefs' }, - userCommitFiles: { ...SCENARIO_DEFAULTS, rate: 2, exec: 'userCommitFiles' }, - userMergeBranch: { ...SCENARIO_DEFAULTS, rate: 1, exec: 'userMergeBranch' }, + // findCommit: { ...SCENARIO_DEFAULTS, rate: 90, exec: 'findCommit' }, + // listCommitsByOid: { ...SCENARIO_DEFAULTS, rate: 90, exec: 'listCommitsByOid' }, + // getBlobs: { ...SCENARIO_DEFAULTS, rate: 90, exec: 'getBlobs' }, + // getTreeEntries: { ...SCENARIO_DEFAULTS, rate: 90, exec: 'getTreeEntries' }, + // treeEntry: { ...SCENARIO_DEFAULTS, rate: 40, exec: 'treeEntry' }, + writeAndDeleteRefs: { ...SCENARIO_DEFAULTS, rate: 40, exec: 'writeAndDeleteRefs' }, + // userCommitFiles: { ...SCENARIO_DEFAULTS, rate: 2, exec: 'userCommitFiles' }, + // userMergeBranch: { ...SCENARIO_DEFAULTS, rate: 1, exec: 'userMergeBranch' }, }, setupTimeout: '5m' } diff --git a/_support/benchmarking/roles/benchmark/tasks/bench.yml b/_support/benchmarking/roles/benchmark/tasks/bench.yml index f9104a1a04422dd25517e0127f5774080bbd652b..992a30f024f9fc7013bc992e065cf905a5451c3c 100644 --- a/_support/benchmarking/roles/benchmark/tasks/bench.yml +++ b/_support/benchmarking/roles/benchmark/tasks/bench.yml @@ -28,6 +28,13 @@ when: clear_page_cache | bool delegate_to: "{{ target_host }}" +# Clear the stacked overlayfs state dir +- name: Clear tacked overlayfs state dir + shell: rm -rf /mnt/git-repositories/stacked-overlayfs + when: clear_stacked_overlayfs_state | bool + delegate_to: "{{ target_host }}" + become: yes + # Create new Gitaly process for each run for easy log collection - name: Start Gitaly service systemd: @@ -69,6 +76,101 @@ owner: git group: git +# Monitor memory usage +- name: Copy memory monitor script + copy: + src: files/memory-monitor.sh + dest: /tmp/memory-monitor.sh + mode: '0755' + delegate_to: "{{ target_host }}" + +- name: Start memory monitoring + shell: | + nohup /tmp/memory-monitor.sh /tmp/{{ run_name }}/memory-usage.log > /dev/null 2>&1 & + echo $! > /tmp/{{ run_name }}-memory-monitor.pid + delegate_to: "{{ target_host }}" + +# Monitor mount count +- name: Start ount count monitoring + shell: | + nohup sh -c 'while true; do + echo "$(date -Iseconds) $(cat /proc/$(pgrep gitaly)/mounts | grep overlay | wc -l)" >> /tmp/{{ run_name }}/mount-count.log + sleep 3 + done' > /dev/null 2>&1 & + echo $! > /tmp/{{ run_name }}-mount-count.pid + delegate_to: "{{ target_host }}" + +# Monitor dentry cache +# /proc/sys/fs/dentry-state — VFS-level view dentry cache +# 387749 382090 45 0 312456 0 +# │ │ │ │ │ │ +# │ │ │ │ │ └── dummy +# │ │ │ │ └── negative dentries (lookups that found nothing) +# │ │ │ └── want_pages (pressure indicator) +# │ │ └── age_limit +# │ └── nr_unused (cached, not actively referenced) +# └── nr_dentry (total dentries in use by VFS) +- name: Start dentry cache monitoring + shell: | + nohup sh -c 'while true; do + echo "$(date -Iseconds) $(cat /proc/sys/fs/dentry-state)" >> /tmp/{{ run_name }}/dentry-state.log + sleep 3 + done' > /dev/null 2>&1 & + echo $! > /tmp/{{ run_name }}-dentry-state.pid + delegate_to: "{{ target_host }}" + +# Monitor Kernel dentry slab memory allocator +# /proc/slabinfo — Kernel memory allocator view +# dentry 388226 388248 192 42 2 ... +# │ │ │ │ │ +# │ │ │ │ └── pages per slab +# │ │ │ └── objects per slab +# │ │ └── object size (bytes) +# │ └── total allocated slots +# └── active objects +- name: Start slab memory monitoring + shell: | + nohup sh -c 'while true; do + echo "$(date -Iseconds) $(cat /proc/slabinfo | grep dentry)" >> /tmp/{{ run_name }}/slab-mem-dentry.log + sleep 3 + done' > /dev/null 2>&1 & + echo $! > /tmp/{{ run_name }}-slab-mem-dentry.pid + delegate_to: "{{ target_host }}" + +# Monitor ovl_inode +# cat /proc/slabinfo | grep ovl +# ovl_inode 1845 1980 720 45 8 : tunables 0 0 0 : slabdata 44 44 0 +# │ │ │ │ │ │ │ │ +# │ │ │ │ │ │ │ └── shared +# │ │ │ │ │ │ └── total slabs +# │ │ │ │ │ └── active slabs +# │ │ │ │ └── pages per slab +# │ │ │ └── objects per slab +# │ │ └── object size (bytes) +# │ └── total allocated objects (slots available) +# └── active objects (actually in use) +- name: Start ovl_inode monitoring + shell: | + nohup sh -c 'while true; do + echo "$(date -Iseconds) $(cat /proc/slabinfo | grep ovl)" >> /tmp/{{ run_name }}/ovl_inode.log + sleep 3 + done' > /dev/null 2>&1 & + echo $! > /tmp/{{ run_name }}-ovl_inode.pid + delegate_to: "{{ target_host }}" + +# Monitor file descriptor +# cat /proc/sys/fs/file-nr +# allocated | unused | max +- name: Start file descriptor monitoring + shell: | + nohup sh -c 'while true; do + echo "$(date -Iseconds) $(cat /proc/sys/fs/file-nr)" >> /tmp/{{ run_name }}/file-descriptor.log + sleep 3 + done' > /dev/null 2>&1 & + echo $! > /tmp/{{ run_name }}-file-descriptor.pid + delegate_to: "{{ target_host }}" + + # We run this task as async and poll 0 to make it non-blocking. # It will be allowed to run up to `workload_wait_duration` before # being treated as failed. @@ -137,4 +239,32 @@ recursive: true delegate_to: "{{ groups['client'][0] }}" # required to copy between two remote hosts +- name: Stop memory monitoring + shell: kill $(cat /tmp/{{ run_name }}-memory-monitor.pid) + ignore_errors: true + delegate_to: "{{ target_host }}" + +- name: Stop mount count monitoring + shell: kill $(cat /tmp/{{ run_name }}-mount-count.pid) + ignore_errors: true + delegate_to: "{{ target_host }}" + +- name: Stop dentry cache monitoring + shell: kill $(cat /tmp/{{ run_name }}-dentry-state.pid) + ignore_errors: true + delegate_to: "{{ target_host }}" + +- name: Stop slab dentry memory monitoring + shell: kill $(cat /tmp/{{ run_name }}-slab-mem-dentry.pid) + ignore_errors: true + delegate_to: "{{ target_host }}" + +- name: Stop ovl inode monitoring + shell: kill $(cat /tmp/{{ run_name }}-ovl_inode.pid) + ignore_errors: true + delegate_to: "{{ target_host }}" +- name: Stop file descriptor monitoring + shell: kill $(cat /tmp/{{ run_name }}-file-descriptor.pid) + ignore_errors: true + delegate_to: "{{ target_host }}" \ No newline at end of file diff --git a/_support/benchmarking/roles/benchmark/vars/main.yml b/_support/benchmarking/roles/benchmark/vars/main.yml index 8d298386d8dd7f1ff9cd2f80bb93576a5f92e0ba..ebb46f4404c10e0dbd044205dadce386668ed51b 100644 --- a/_support/benchmarking/roles/benchmark/vars/main.yml +++ b/_support/benchmarking/roles/benchmark/vars/main.yml @@ -1,6 +1,7 @@ --- profile: true clear_page_cache: true +clear_stacked_overlayfs_state: true # Profiling and the workload will begin concurrently. profile_duration: 120 workload_duration: "120s" diff --git a/_support/benchmarking/roles/gitaly/tasks/initialize.yml b/_support/benchmarking/roles/gitaly/tasks/initialize.yml index ae4848907655bed3839899163be1afd7eca0d777..4942a986d9d26cab5994410577288d4975f2713a 100644 --- a/_support/benchmarking/roles/gitaly/tasks/initialize.yml +++ b/_support/benchmarking/roles/gitaly/tasks/initialize.yml @@ -117,6 +117,28 @@ args: chdir: /mnt/git-repositories executable: /bin/bash +- name: Create random large number of small files + shell: | + for i in $(seq 1 {{ item.random_small_file_number | default(0) }}); do + dd if=/dev/urandom of=/mnt/git-repositories/{{ item.name }}/rad_file_$i.dat bs=1K count=1 2>/dev/null + done + loop: "{{ config.repositories | default([]) }}" + when: + - config.create_random_small_files | default(false) + - item.random_small_file_number | default(0) > 0 + ignore_errors: yes + +- name: Duplicate repositories for overlayfs testing + shell: | + for i in $(seq 1 {{ item.duplicate_counter | default(0) }}); do + dest="/mnt/git-repositories/{{ item.name }}$i" + if [ ! -d "$dest" ]; then + echo "Duplicating {{ item.name }} -> $dest" + cp -a "/mnt/git-repositories/{{ item.name }}" "$dest" + fi + done + loop: "{{ config.repositories | default([]) }}" + when: config.duplicate_repo_for_overlayfs | default(false) - name: Set git as owner of repositories directory file: diff --git a/_support/benchmarking/roles/gitaly/templates/config.toml.j2 b/_support/benchmarking/roles/gitaly/templates/config.toml.j2 index 2cbc82c878389754053aa0ff03d2da99e2645bcc..0fcfe48e82d6deebb8bea2a1ae3e2e415b7ee701 100644 --- a/_support/benchmarking/roles/gitaly/templates/config.toml.j2 +++ b/_support/benchmarking/roles/gitaly/templates/config.toml.j2 @@ -51,6 +51,7 @@ max_age = '5m' [transactions] enabled = {{ target_instance['config']['transactions'] | default(false) | lower }} +driver = "{{ target_instance['config']['snapshot_driver'] | default("deepclone") | lower }}" [raft] enabled = {{ target_instance['config']['raft'] | default(false) | lower }} \ No newline at end of file diff --git a/_support/benchmarking/roles/gitaly/templates/gitaly.service.j2 b/_support/benchmarking/roles/gitaly/templates/gitaly.service.j2 index e7945fb33ac60ffc5e257c8f3c76190bfac9ecc1..e41b91c97baafdfa5da8253cd50f2b6c0ab38f16 100644 --- a/_support/benchmarking/roles/gitaly/templates/gitaly.service.j2 +++ b/_support/benchmarking/roles/gitaly/templates/gitaly.service.j2 @@ -4,7 +4,7 @@ After=syslog.target network.target remote-fs.target nss-lookup.target mock-gitla Requires=mock-gitlab-api.service [Service] -ExecStart=/opt/gitaly/bin/gitaly /var/opt/gitaly/config.toml +ExecStart=/usr/bin/unshare --user --mount --map-root-user /opt/gitaly/bin/gitaly /var/opt/gitaly/config.toml ExecReload=/bin/kill -s HUP $MAINPID ExecStop=/bin/kill -s TERM $MAINPID WorkingDirectory=/var/opt/gitaly