From 1d2cc30dda11e80031bd67b5c69215d558db299c Mon Sep 17 00:00:00 2001 From: Nailia Iskhakova Date: Tue, 22 Jun 2021 22:14:46 +0300 Subject: [PATCH 1/4] Add 50k hybrid RA chart example https://gitlab.com/gitlab-org/quality/reference-architectures/issues/24 Signed-off-by: Nailia Iskhakova --- examples/ref/50k.yaml | 193 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 193 insertions(+) create mode 100644 examples/ref/50k.yaml diff --git a/examples/ref/50k.yaml b/examples/ref/50k.yaml new file mode 100644 index 0000000000..0b9e6d564e --- /dev/null +++ b/examples/ref/50k.yaml @@ -0,0 +1,193 @@ +gitlab: + # n1-highcpu-32 in GKE is + # - cpu: 31850m + # - memory: 26041340Ki + webservice: + # Min pods recommended to be set to 75% of max to start + maxReplicas: 80 + maxUnavailable: 40 + minReplicas: 60 + hpa: + targetAverageValue: 1600m + ingress: + proxyBodySize: 0 # To allow large file uploads like imports + nodeSelector: + workload: webservice + resources: + limits: + memory: 5.25G # roughly, 1.3GB/worker + requests: + cpu: 4 # requests.cpu <= workerProcesses + memory: 5G # roughly, 1.25GB/worker + workerProcesses: 4 + sidekiq: + maxReplicas: 14 + minReplicas: 10 + hpa: + targetAverageValue: 700m + nodeSelector: + workload: sidekiq + resources: + # Running up to 16 pods with a ratio of roughly 1 vCPU to 2 GB RAM as a starting point for requests. + # Per the 10k reference architecture (https://docs.gitlab.com/ee/administration/reference_architectures/10k_users.html), + # consider the resource limitations in play: + # For an n1-standard-4 node in GKE: 4 vCPU, 15 GB memory + # - 4 nodes x 4 vCPU = 16 cores in total + # - 4 nodes x 15 GB = 60GB of memory in total + # - 60GB memory / 16 cores = ~3.75 GB memory / core, excluding overhead + # Each Sidekiq node has the following resources allocatable (roughly): + # - cpu: 3920m + # - memory: 12698332Ki + # Notice that there is actually only <4 vCPU and <13GB memory allocatable. + limits: + memory: 4G + requests: + cpu: 900m # Assume single-process, 1 CPU + memory: 2G + task-runner: + backups: + # Below, GCS is used an example for the ObjectStorage section. + # All ObjectStorage options are documented here: + # https://docs.gitlab.com/ee/administration/reference_architectures/10k_users.html#configure-the-object-storage + objectStorage: + backend: gcs + config: + gcpProject: example-gcp-projectname + key: gcs.json + secret: example-connection-secret + migrations: + # Migrations pod must point directly to PostgreSQL primary + psql: + host: example-postgresql-primary-ip + password: + secret: example-postgresql-secret + key: password + port: example-postgresql-primary-port + +global: + appConfig: + # For context on the ObjectStorage configuration below, see: + # https://docs.gitlab.com/ee/administration/reference_architectures/10k_users.html#configure-the-object-storage + object_store: + enabled: true + connection: + secret: example-storage-secret + key: password + artifacts: + bucket: example-artifacts-bucket-name + backups: + bucket: example-backups-bucket-name + dependencyProxy: + bucket: example-dependency-proxy-bucket-name + externalDiffs: + bucket: example-external-diffs-bucket-name + # Recommend to only store older External Diffs in Object Storage for performance reasons + when: outdated + lfs: + bucket: example-lfs-bucket-name + packages: + bucket: example-packages-bucket-name + terraformState: + bucket: example-terraform-state-bucket-name + uploads: + bucket: example-uploads-bucket-name + shell: + authToken: + secret: example-shell-secret + key: password + gitaly: + authToken: + secret: example-gitaly-praefect-external-token + key: password + enabled: false # Run Gitaly outside of cluster + external: + - name: default + hostname: default.praefect.gitaly.com + port: example-praefect-port + gitlab: + license: + secret: example-gitlab-license + grafana: + enabled: true + hosts: + domain: example.com + minio: + enabled: false + psql: + host: db.example.com + password: + secret: example-postgresql-secret + key: password + port: example-postgresql-port + redis: + cache: + host: example-redis-cache-1 + password: + key: password + secret: example-redis-secret + sentinels: + - host: cache.sentinel1.example.com + - host: cache.sentinel2.example.com + - host: cache.sentinel3.example.com + host: example-redis-queue-1 + password: + key: password + secret: example-redis-secret + sentinels: + - host: sentinel1.example.com + - host: sentinel2.example.com + - host: sentinel3.example.com + registry: + bucket: example-registry-storage + +# https://docs.gitlab.com/ee/ci/runners/#configuring-runners-in-gitlab +gitlab-runner: + install: false + +# Below, we use nodeSelectors to send certain +# workloads to a node separate from the nodes +# used for the Webservice and Sidekiq workloads. +# This helps to ensure Webservice and Sidekiq have +# sufficient resources to schedule enough pods to +# meet the 10k reference architecture specifications. +grafana: + nodeSelector: + workload: "" +nginx-ingress: + controller: + nodeSelector: + workload: "" + defaultBackend: + nodeSelector: + workload: "" +prometheus: + alertmanager: + nodeSelector: + workload: "" + install: true + kubeStateMetrics: + nodeSelector: + workload: "" + nodeExporter: + nodeSelector: + workload: "" + pushgateway: + nodeSelector: + workload: "" + server: + nodeSelector: + workload: "" + +# Run Redis outside of cluster +redis: + install: false + +registry: + storage: + extraKey: gcs.json + key: registry-config + secret: gcs-secret-example + +# Database run outside of cluster +postgresql: + install: false -- GitLab From 2b80fe6b96085cc5f70274544d291d75bac9bc82 Mon Sep 17 00:00:00 2001 From: Nailia Iskhakova Date: Wed, 23 Jun 2021 16:19:03 +0300 Subject: [PATCH 2/4] Adjust sidekiq pods count in 10k hybrid comments Signed-off-by: Nailia Iskhakova --- examples/ref/10k.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ref/10k.yaml b/examples/ref/10k.yaml index 6c014b4585..de711a87d5 100644 --- a/examples/ref/10k.yaml +++ b/examples/ref/10k.yaml @@ -28,7 +28,7 @@ gitlab: nodeSelector: workload: sidekiq resources: - # Running up to 16 pods with a ratio of roughly 1 vCPU to 2 GB RAM as a starting point for requests. + # Running up to 14 pods with a ratio of roughly 1 vCPU to 2 GB RAM as a starting point for requests. # Per the 10k reference architecture (https://docs.gitlab.com/ee/administration/reference_architectures/10k_users.html), # consider the resource limitations in play: # For an n1-standard-4 node in GKE: 4 vCPU, 15 GB memory -- GitLab From a6ecf5ec9508679b8a1ab3e7bfbdc263da40dd80 Mon Sep 17 00:00:00 2001 From: Nailia Iskhakova Date: Wed, 23 Jun 2021 16:24:12 +0300 Subject: [PATCH 3/4] Update sidekiq comment in 50k as well Signed-off-by: Nailia Iskhakova --- examples/ref/50k.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/ref/50k.yaml b/examples/ref/50k.yaml index 0b9e6d564e..da7e1c810c 100644 --- a/examples/ref/50k.yaml +++ b/examples/ref/50k.yaml @@ -28,7 +28,7 @@ gitlab: nodeSelector: workload: sidekiq resources: - # Running up to 16 pods with a ratio of roughly 1 vCPU to 2 GB RAM as a starting point for requests. + # Running up to 14 pods with a ratio of roughly 1 vCPU to 2 GB RAM as a starting point for requests. # Per the 10k reference architecture (https://docs.gitlab.com/ee/administration/reference_architectures/10k_users.html), # consider the resource limitations in play: # For an n1-standard-4 node in GKE: 4 vCPU, 15 GB memory -- GitLab From 667a3e1d16f76ecdbe8ccc4e3a1bcff6800cfef0 Mon Sep 17 00:00:00 2001 From: Nailia Iskhakova Date: Wed, 23 Jun 2021 16:26:10 +0300 Subject: [PATCH 4/4] Refer to 50k reference architecture docs link Signed-off-by: Nailia Iskhakova --- examples/ref/50k.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/examples/ref/50k.yaml b/examples/ref/50k.yaml index da7e1c810c..1ac736b5df 100644 --- a/examples/ref/50k.yaml +++ b/examples/ref/50k.yaml @@ -29,7 +29,7 @@ gitlab: workload: sidekiq resources: # Running up to 14 pods with a ratio of roughly 1 vCPU to 2 GB RAM as a starting point for requests. - # Per the 10k reference architecture (https://docs.gitlab.com/ee/administration/reference_architectures/10k_users.html), + # Per the 50k reference architecture (https://docs.gitlab.com/ee/administration/reference_architectures/50k_users.html), # consider the resource limitations in play: # For an n1-standard-4 node in GKE: 4 vCPU, 15 GB memory # - 4 nodes x 4 vCPU = 16 cores in total @@ -48,7 +48,7 @@ gitlab: backups: # Below, GCS is used an example for the ObjectStorage section. # All ObjectStorage options are documented here: - # https://docs.gitlab.com/ee/administration/reference_architectures/10k_users.html#configure-the-object-storage + # https://docs.gitlab.com/ee/administration/reference_architectures/50k_users.html#configure-the-object-storage objectStorage: backend: gcs config: @@ -67,7 +67,7 @@ gitlab: global: appConfig: # For context on the ObjectStorage configuration below, see: - # https://docs.gitlab.com/ee/administration/reference_architectures/10k_users.html#configure-the-object-storage + # https://docs.gitlab.com/ee/administration/reference_architectures/50k_users.html#configure-the-object-storage object_store: enabled: true connection: @@ -149,7 +149,7 @@ gitlab-runner: # used for the Webservice and Sidekiq workloads. # This helps to ensure Webservice and Sidekiq have # sufficient resources to schedule enough pods to -# meet the 10k reference architecture specifications. +# meet the 50k reference architecture specifications. grafana: nodeSelector: workload: "" -- GitLab