diff --git a/k8s-supplements/ansible/roles/monitoring_thanos_v2/defaults/main.yaml b/k8s-supplements/ansible/roles/monitoring_thanos_v2/defaults/main.yaml index 4efbd897cce0d45e6ab8f0f0909f4e016d49e020..37cd45ecaf3fca34abada01de1eb4f1386d6eebe 100644 --- a/k8s-supplements/ansible/roles/monitoring_thanos_v2/defaults/main.yaml +++ b/k8s-supplements/ansible/roles/monitoring_thanos_v2/defaults/main.yaml @@ -5,11 +5,5 @@ monitoring_thanos_helm_oci_url: "oci://registry-1.docker.io/bitnamicharts" monitoring_thanos_chart_ref: "thanos" monitoring_thanos_release_name: "thanos" -monitoring_thanos_config_secret_name: "thanos-bucket-config" - -monitoring_thanos_retention_resolution_raw: "30d" -monitoring_thanos_retention_resolution_5m: "60d" -monitoring_thanos_retention_resolution_1h: "180d" - monitoring_common_labels: {} ... diff --git a/k8s-supplements/ansible/roles/monitoring_thanos_v2/tasks/main.yaml b/k8s-supplements/ansible/roles/monitoring_thanos_v2/tasks/main.yaml index 52d843e34546ceeebea21e05af04d2308bf8579a..683ee42dcc1df33a306c1a10088f97d5ce6a16ef 100644 --- a/k8s-supplements/ansible/roles/monitoring_thanos_v2/tasks/main.yaml +++ b/k8s-supplements/ansible/roles/monitoring_thanos_v2/tasks/main.yaml @@ -95,15 +95,13 @@ # For helm charts of Bitnami the OCI format must be used # https://community.broadcom.com/tanzu/blogs/carlos-rodriguez-hernandez/2025/01/14/bitnami-helm-charts-moving-to-oci - name: "{{ monitoring_use_thanos | ternary('I', 'Uni') }}nstall Thanos helm chart" - vars: - scheduling_key: "{{ monitoring_scheduling_key | default(None) }}" kubernetes.core.helm: chart_ref: "{{ monitoring_thanos_helm_oci_url }}/{{ monitoring_thanos_chart_ref }}" chart_version: "{{ monitoring_thanos_chart_version }}" release_namespace: "{{ monitoring_namespace }}" release_name: "{{ monitoring_thanos_release_name }}" release_state: "{{ monitoring_use_thanos | ternary('present', 'absent') }}" - values: "{{ lookup('template', 'thanos_values.yaml.j2') | from_yaml }}" + values: "{{ monitoring_thanos_values | to_json }}" wait: true tags: - thanos diff --git a/k8s-supplements/ansible/roles/monitoring_thanos_v2/templates/thanos_values.yaml.j2 b/k8s-supplements/ansible/roles/monitoring_thanos_v2/templates/thanos_values.yaml.j2 deleted file mode 100644 index 7c0b424236d62b256abd380a927ad53225250835..0000000000000000000000000000000000000000 --- a/k8s-supplements/ansible/roles/monitoring_thanos_v2/templates/thanos_values.yaml.j2 +++ /dev/null @@ -1,76 +0,0 @@ ---- -global: -{% if monitoring_thanos_storage_class is not none %} - storageClass: {{ monitoring_thanos_storage_class | to_json }} -{% endif %} -{# Original containers have been substituted, see https://github.com/bitnami/containers/issues/83267 #} - security: - allowInsecureImages: true -image: - repository: bitnamilegacy/thanos - -existingObjstoreSecret: {{ monitoring_thanos_config_secret_name | quote }} -compactor: - enabled: true - extraFlags: - - "--no-debug.halt-on-error" # must exit for Kubernetes to restart it - retentionResolutionRaw: {{ monitoring_thanos_retention_resolution_raw | quote }} - retentionResolution5m: {{ monitoring_thanos_retention_resolution_5m | quote }} - retentionResolution1h: {{ monitoring_thanos_retention_resolution_1h | quote }} - resources: {{ monitoring_thanos_compact_resources | to_json }} - tolerations: {{ lookup('template', 'roles/config/common_defaults_v1/templates/tolerations.json.j2') | to_json }} - affinity: {{ lookup('template', 'roles/config/common_defaults_v1/templates/affinity.json.j2') | to_json }} - persistence: - enabled: {{ monitoring_thanos_storage_class is not none }} -{% if monitoring_thanos_compactor_size is not none %} - size: {{ monitoring_thanos_compactor_size | to_json }} -{% endif %} - -storegateway: - enabled: true -{% if monitoring_thanos_store_in_memory_max_size is not none %} - extraFlags: - - "--index-cache-size={{ monitoring_thanos_store_in_memory_max_size }}" -{% endif %} - resources: {{ monitoring_thanos_store_resources | to_json }} - tolerations: {{ lookup('template', 'roles/config/common_defaults_v1/templates/tolerations.json.j2') | to_json }} - affinity: {{ lookup('template', 'roles/config/common_defaults_v1/templates/affinity.json.j2') | to_json }} - pdb: - create: false - replicaCount: 1 - persistence: - enabled: {{ monitoring_thanos_storage_class is not none }} -{% if monitoring_thanos_storegateway_size is not none %} - size: {{ monitoring_thanos_storegateway_size | to_json }} -{% endif %} - -query: - enabled: true - resources: {{ monitoring_thanos_query_resources | to_json }} - dnsDiscovery: - enabled: true - sidecarsService: "prometheus-operated" - sidecarsNamespace: "{{ monitoring_namespace }}" - affinity: {{ lookup('template', 'roles/config/common_defaults_v1/templates/affinity.json.j2') | to_json }} - tolerations: {{ lookup('template', 'roles/config/common_defaults_v1/templates/tolerations.json.j2') | to_json }} - pdb: - create: false - replicaCount: 1 - extraFlags: - - "--query.auto-downsampling" - - "--query.timeout=1m" - -queryFrontend: - enabled: false - -metrics: - enabled: true - serviceMonitor: - enabled: true -{% if monitoring_common_labels %} - labels: -{% for label_key, label_value in monitoring_common_labels.items() %} - {{ label_key | to_json }}: {{ label_value | to_json }} -{% endfor %} -{% endif %} -... diff --git a/k8s-supplements/ansible/roles/monitoring_v2/tasks/main.yaml b/k8s-supplements/ansible/roles/monitoring_v2/tasks/main.yaml index 4e1631c1da572697c97fa20dd887471a14f8c51f..15975f66180a961a4fd32ef45ac20490ec6c6d6c 100644 --- a/k8s-supplements/ansible/roles/monitoring_v2/tasks/main.yaml +++ b/k8s-supplements/ansible/roles/monitoring_v2/tasks/main.yaml @@ -91,8 +91,6 @@ # Install the prometheus stack - name: "{{ monitoring_install | ternary('I', 'Uni') }}nstall KSL monitoring stack" - vars: - scheduling_key: "{{ monitoring_scheduling_key | default(None) }}" kubernetes.core.helm: chart_repo_url: "{{ monitoring_prometheus_helm_repo_url }}" chart_ref: "{{ monitoring_prometheus_stack_chart_name }}" @@ -100,7 +98,7 @@ release_namespace: "{{ monitoring_namespace }}" release_name: "{{ monitoring_prometheus_stack_release_name }}" release_state: "{{ monitoring_install | ternary('present', 'absent') }}" - values: "{{ lookup('template', 'prometheus_stack.yaml.j2') | from_yaml }}" + values: "{{ monitoring_prometheus_stack_values | to_json }}" wait: false # Install prometheus-adapter, a metrics-server implementation diff --git a/k8s-supplements/ansible/roles/monitoring_v2/templates/prometheus_stack.yaml.j2 b/k8s-supplements/ansible/roles/monitoring_v2/templates/prometheus_stack.yaml.j2 deleted file mode 100644 index 44fdbef61b1e37bb7a90d74501cf5a9dc6bbe55a..0000000000000000000000000000000000000000 --- a/k8s-supplements/ansible/roles/monitoring_v2/templates/prometheus_stack.yaml.j2 +++ /dev/null @@ -1,444 +0,0 @@ -## Labels to apply to all resources -## -{% if monitoring_common_labels %} -commonLabels: -{% for label_key, label_value in monitoring_common_labels.items() %} - {{ label_key | to_json }}: {{ label_value | to_json }} -{% endfor %} -{% endif %} - -priorityClassName: "system-cluster-critical" - -## -defaultRules: - create: true - rules: - etcd: false # disabled for now - kubeApiserver: false # https://github.com/prometheus-community/helm-charts/issues/1283 - -## -global: - rbac: - create: true - -## -alertmanager: - enabled: true - alertmanagerSpec: - priorityClassName: "system-cluster-critical" - replicas: "{{ monitoring_alertmanager_replicas }}" -{% if monitoring_allow_external_rules %} - alertmanagerConfigMatcherStrategy: - type: None -{% endif %} - serviceMonitor: - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - affinity: {{ lookup('template', 'roles/config/common_defaults_v1/templates/affinity.json.j2') | to_json }} - tolerations: {{ lookup('template', 'roles/config/common_defaults_v1/templates/tolerations.json.j2') | to_json }} - -## -grafana: - priorityClassName: "system-cluster-critical" - enabled: {{ monitoring_use_grafana | bool }} - persistence: - enabled: {{ monitoring_grafana_persistent_storage_class is not none }} - storageClassName: "{{ monitoring_grafana_persistent_storage_class | default("") }}" - admin: - existingSecret: {{ monitoring_grafana_admin_secret_name }} - userKey: admin-user - passwordKey: admin-password - resources: {{ monitoring_grafana_resources | to_json }} - tolerations: {{ lookup('template', 'roles/config/common_defaults_v1/templates/tolerations.json.j2') | to_json }} -{% if monitoring_grafana_persistent_storage_class is not none %} - affinity: {{ lookup('template', 'roles/config/common_defaults_v1/templates/affinity.json.j2', template_vars=dict(pod_affinity_key="app.kubernetes.io/name", pod_affinity_operator="In", pod_affinity_values=["grafana"])) | to_json }} -{% else %} - affinity: {{ lookup('template', 'roles/config/common_defaults_v1/templates/affinity.json.j2') | to_json }} -{% endif %} - datasources: - datasources.yaml: - apiVersion: 1 -{% if monitoring_use_thanos %} - datasources: - - name: thanos - type: prometheus - access: proxy - orgId: 1 - url: "http://thanos-query.{{ monitoring_namespace }}.svc:9090" - version: 1 - editable: false -{% endif %} - sidecar: - dashboards: - enabled: true - label: grafana_dashboard - searchNamespace: "ALL" - folderAnnotation: customer-dashboards - provider: - foldersFromFilesStructure: true -{% if monitoring_grafana_dashboard_enable_multicluster_support | bool %} - multicluster: - global: - enabled: true -{% endif %} - serviceMonitor: - enabled: true -{% if monitoring_common_labels %} - labels: -{% for label_key, label_value in monitoring_common_labels.items() %} - {{ label_key | to_json }}: {{ label_value | to_json }} -{% endfor %} -{% endif %} - # https://github.com/prometheus-community/helm-charts/issues/1776 - interval: "30s" - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - dashboards: - managed: - dashboard-calico: - gnetId: 12175 - revision: 5 - datasource: Prometheus -{% if rook_enabled %} - dashboard-ceph-cluster: - gnetId: 2842 - revision: 14 - datasource: Prometheus - dashboard-ceph-osd-single: - gnetId: 5336 - revision: 5 - datasource: Prometheus - dashboard-ceph-pools: - gnetId: 5342 - revision: 5 - datasource: Prometheus -{% endif %} -{% if monitoring_use_thanos %} - dashboard-thanos: - gnetId: 12937 - revision: 4 - datasource: Prometheus -{% endif %} -{% if k8s_ingress_enabled | bool %} - dashboard-nginx-ingress: - gnetId: 9614 - revision: 1 - datasource: Prometheus -{% endif %} - dashboardProviders: - managed-dashboard-provider.yaml: - apiVersion: 1 - providers: - - name: 'managed-dashboards' - folder: 'managed-dashboards' - options: - path: /var/lib/grafana/dashboards/managed - grafana.ini: - server: -{% if monitoring_grafana_root_url is not none %} - root_url: {{ monitoring_grafana_root_url | to_json }} -{% endif %} - -## -kubeApiServer: - enabled: true - serviceMonitor: - relabelings: - - sourceLabels: - - __meta_kubernetes_namespace - - __meta_kubernetes_service_name - - __meta_kubernetes_endpoint_port_name - action: keep - regex: default;kubernetes;https - - targetLabel: __address__ - replacement: kubernetes.default.svc:443 - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - -## -kubelet: - enabled: true - serviceMonitor: - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - - sourceLabels: [__metrics_path__] - targetLabel: metrics_path - action: replace - -## -kubeControllerManager: - enabled: true - service: - port: 10257 - targetPort: 10257 - serviceMonitor: - enabled: true - https: true - insecureSkipVerify: true - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - -## -coreDNS: - enabled: true - serviceMonitor: - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - -## -kubeEtcd: - enabled: true - service: - enabled: true - port: 2381 - targetPort: 12381 - selector: - app.kubernetes.io/name: etcd-proxy-metrics - ipDualStack: - enabled: {{ ipv4_enabled and ipv6_enabled | ternary(true, false) }} - ipFamilies: -{% if ipv4_enabled %} - - IPv4 -{% endif %} -{% if ipv6_enabled %} - - IPv6 -{% endif %} - ipFamilyPolicy: {{ ipv4_enabled and ipv6_enabled | ternary("PreferDualStack", "SingleStack") }} - serviceMonitor: - enabled: true - scheme: https - insecureSkipVerify: false - caFile: /etc/prometheus/secrets/etcd-metrics-proxy/server.crt - certFile: /etc/prometheus/secrets/etcd-metrics-proxy/client.crt - keyFile: /etc/prometheus/secrets/etcd-metrics-proxy/client.key - -## -kubeScheduler: - enabled: true - service: - enabled: true - port: 10259 - targetPort: 10259 - serviceMonitor: - enabled: true - https: true - insecureSkipVerify: true - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - -## -kubeProxy: - enabled: true - serviceMonitor: - enabled: true - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - -## -kubeStateMetrics: - enabled: true - serviceMonitor: - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - -## -kube-state-metrics: - priorityClassName: "system-cluster-critical" - rbac: - create: true - pspEnabled: false -{% if monitoring_common_labels %} - customLabels: -{% for label_key, label_value in monitoring_common_labels.items() %} - {{ label_key | to_json }}: {{ label_value | to_json }} -{% endfor %} -{% endif %} - metricLabelsAllowlist: - - namespaces=[*] - -## -nodeExporter: - enabled: true - ## Use the value configured in prometheus-node-exporter.podLabels - jobLabel: jobLabel - -## Configuration for prometheus-node-exporter subchart -## -prometheus-node-exporter: - priorityClassName: "system-node-critical" - namespaceOverride: "" - podLabels: - ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards - ## - jobLabel: node-exporter - extraArgs: - - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/) - - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ - prometheus: - monitor: - enabled: true -{% if monitoring_common_labels %} - additionalLabels: -{% for label_key, label_value in monitoring_common_labels.items() %} - {{ label_key | to_json }}: {{ label_value | to_json }} -{% endfor %} -{% endif %} - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - - sourceLabels: [__meta_kubernetes_node_labelpresent_node_role_kubernetes_io_control_plane] - targetLabel: role - regex: "^true$" - replacement: control-plane - - sourceLabels: [__meta_kubernetes_node_labelpresent_node_role_kubernetes_io_worker] - targetLabel: role - regex: "^true$" - replacement: worker - attachMetadata: - node: true - -## -prometheusOperator: - enabled: true - priorityClassName: "system-cluster-critical" - admissionWebhooks: - patch: - priorityClassName: "system-cluster-critical" - resources: {{ monitoring_operator_resources | to_json }} - serviceMonitor: - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - affinity: {{ lookup('template', 'roles/config/common_defaults_v1/templates/affinity.json.j2') | to_json }} - tolerations: {{ lookup('template', 'roles/config/common_defaults_v1/templates/tolerations.json.j2') | to_json }} - -## -prometheus: - enabled: true - thanosService: - enabled: {{ monitoring_use_thanos | bool }} - thanosServiceMonitor: - enabled: {{ monitoring_use_thanos | bool }} - serviceMonitor: - relabelings: - - sourceLabels: [__meta_kubernetes_pod_node_name] - separator: ; - regex: ^(.*)$ - targetLabel: nodename - replacement: $1 - action: replace - prometheusSpec: - priorityClassName: "system-cluster-critical" -{% if monitoring_remote_writes | length > 0 %} - remoteWrite: -{% for remote_write in monitoring_remote_writes %} - - url: {{ remote_write.url | to_json }} - writeRelabelConfigs: {{ remote_write.write_relabel_configs | to_json }} -{% if remote_write.basic_auth_secret_name is not none %} - basicAuth: - username: - name: {{ remote_write.basic_auth_secret_name | to_json }} - key: username - password: - name: {{ remote_write.basic_auth_secret_name | to_json }} - key: password -{% endif %} -{% endfor %} -{% endif %} - secrets: - - etcd-metrics-proxy - serviceMonitorSelectorNilUsesHelmValues: {{ monitoring_common_labels | ternary(true, false) }} -{% if monitoring_prometheus_persistent_storage_class is not none %} - storageSpec: - volumeClaimTemplate: - spec: - storageClassName: {{ monitoring_prometheus_persistent_storage_class | to_json }} - accessModes: ["ReadWriteOnce"] - resources: - requests: - storage: {{ monitoring_prometheus_persistent_storage_resource_request | to_json }} -{% endif %} -{% if monitoring_common_labels %} - serviceMonitorSelector: - matchLabels: -{% for label_key, label_value in monitoring_common_labels.items() %} - {{ label_key | to_json }}: {{ label_value | to_json }} -{% endfor %} -{% endif %} -{% if monitoring_use_thanos | bool %} - thanos: - objectStorageConfig: - optional: false -{% if monitoring_prometheus_stack_version is version('51.0.0', '>=') %} - existingSecret: - name: thanos-sidecar-bucket-credentials-config - key: thanos.yaml -{% else %} - name: thanos-sidecar-bucket-credentials-config - key: thanos.yaml -{% endif %} -{% endif %} - containers: - - name: prometheus - readinessProbe: - failureThreshold: 1000 - resources: {{ monitoring_prometheus_resources | to_json }} - affinity: {{ lookup('template', 'roles/config/common_defaults_v1/templates/affinity.json.j2') | to_json }} - tolerations: {{ lookup('template', 'roles/config/common_defaults_v1/templates/tolerations.json.j2') | to_json }} -{% if monitoring_allow_external_rules %} - ruleSelectorNilUsesHelmValues: false - ruleSelector: {} - ruleNamespaceSelector: {} -{% endif %} diff --git a/nix/yk8s/k8s-supplements/ingress.nix b/nix/yk8s/k8s-supplements/ingress.nix index e2c74a5994b7edd016c2a464d764eccb4e965131..3356ad8301f80cb0cc346cf5d8eb9d56ac115f5b 100644 --- a/nix/yk8s/k8s-supplements/ingress.nix +++ b/nix/yk8s/k8s-supplements/ingress.nix @@ -9,7 +9,7 @@ inherit (modules-lib) mkRenamedOptionModule mkResourceOptionModule; inherit (lib) mkEnableOption mkOption types; inherit (yk8s-lib) mkTopSection mkGroupVarsFile; - inherit (yk8s-lib.k8s) mkAffinities mkTolerations; + inherit (yk8s-lib.k8s) mkAffinity mkTolerations; inherit (yk8s-lib.options) mkHelmReleaseOptions; inherit (yk8s-lib.types) @@ -139,7 +139,7 @@ in { }; config.yk8s.k8s-service-layer.ingress.helm.values = let inherit (config.yk8s.infra) ipv4_enabled ipv6_enabled; - affinity = mkAffinities {inherit (cfg) scheduling_key;}; + affinity = mkAffinity {inherit (cfg) scheduling_key;}; tolerations = mkTolerations {inherit (cfg) scheduling_key;}; in { defaultBackend = {inherit affinity tolerations;}; diff --git a/nix/yk8s/k8s-supplements/monitoring.nix b/nix/yk8s/k8s-supplements/monitoring/default.nix similarity index 61% rename from nix/yk8s/k8s-supplements/monitoring.nix rename to nix/yk8s/k8s-supplements/monitoring/default.nix index 21ed5831519c5a166a5ba2cfff0c62c4ca79f5e9..b5c495cf6bf01f45df8898abee8ec8bb6b0f75b5 100644 --- a/nix/yk8s/k8s-supplements/monitoring.nix +++ b/nix/yk8s/k8s-supplements/monitoring/default.nix @@ -5,20 +5,17 @@ ... }: let cfg = config.yk8s.k8s-service-layer.prometheus; - modules-lib = import ../lib/modules.nix {inherit lib;}; + modules-lib = import ../../lib/modules.nix {inherit lib;}; inherit (modules-lib) mkRenamedOptionModule mkRemovedOptionModule mkRenamedResourceOptionModule mkMultiResourceOptionsModule; inherit (lib) mkEnableOption mkOption types; - inherit (lib.attrsets) foldlAttrs; - inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkMultiResourceOptions; + inherit (yk8s-lib) mkTopSection mkGroupVarsFile; inherit (yk8s-lib.options) mkHelmChartVersionOption; inherit (yk8s-lib.types) absolutePosixPath - bytesPower10 helmChartReleaseName helmChartRepoUrl helmChartRef - httpxHostPathUrl httpxUrl ipv4Addr ipv4AddrWithPort @@ -32,27 +29,23 @@ k8sSecretName k8sServiceName k8sStorageClassName - openstackSwiftContainerName prometheusIntervalStr prometheusTimeoutStr prometheusRelabelConfig - relativePosixPath - subdomainLabel ; in { imports = [ + ./grafana.nix + ./thanos.nix + ./helm_prometheus_stack + (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "use_jsonnet_setup"] "") - (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "use_helm_thanos"] "") (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "migrate_from_v1"] "") (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "alertmanager_config_secret"] "") (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "alertmanager_configuration_name"] "") (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "kube_state_metrics_metric_annotation_allow_list"] "") - (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "thanos_metadata_volume_size"] "") - (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "thanos_metadata_volume_storage_class"] "") - (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "grafana_plugins"] "") (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "prometheus_monitor_all_namespaces"] "") (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "monitor_all_namespaces"] "") - (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "thanos_objectstorage_config_path"] "Use `thanos_objectstorage_config_file` instead.") (mkRenamedOptionModule ["k8s-service-layer" "prometheus" "prometheus_operator_cpu_request"] ["k8s-service-layer" "prometheus" "operator_resources" "cpu" "request"]) (mkRenamedOptionModule ["k8s-service-layer" "prometheus" "prometheus_operator_cpu_limit"] ["k8s-service-layer" "prometheus" "operator_resources" "cpu" "limit"]) @@ -79,109 +72,20 @@ in { prometheus.memory.limit = "3Gi"; prometheus.cpu.request = "1"; - grafana.memory.limit = "512Mi"; - grafana.cpu.request = "100m"; - grafana.cpu.example = "500m"; - kube_state_metrics.memory.limit = "128Mi"; kube_state_metrics.cpu.request = "50m"; - - thanos_sidecar.memory.limit = "256Mi"; - thanos_sidecar.cpu.request = "500m"; - - thanos_query.memory.limit = "786Mi"; - thanos_query.cpu.request = "100m"; - thanos_query.cpu.example = "1"; - - thanos_compact.memory.limit = "200Mi"; - thanos_compact.cpu.request = "100m"; - - thanos_store.memory.limit = "2Gi"; - thanos_store.cpu.request = "100m"; - thanos_store.cpu.example = "500m"; }; }) (mkRenamedResourceOptionModule ["k8s-service-layer" "prometheus"] [ "operator" "alertmanager" "prometheus" - "grafana" "kube_state_metrics" - "thanos_sidecar" - "thanos_query" - "thanos_compact" - "thanos_store" ]) ]; options.yk8s.k8s-service-layer.prometheus = mkTopSection { - _docs.preface = '' - The used prometheus-based monitoring setup will be explained in more - detail soon :) - - .. note:: - - To enable prometheus, - :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.install` - and - :ref:`configuration-options.yk8s.kubernetes.monitoring.enabled` - need to be set to ``true``. - - - Tweak Thanos Configuration - """""""""""""""""""""""""" - - index-cache-size / in-memory-max-size - ************************************* - - Thanos is unaware of its Kubernetes limits - which can lead to OOM kills of the storegateway - if a lot of metrics are requested. - - This can be prevented by tuning the following config options: - - - :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_store_in_memory_max_size` - - :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_store_resources.limits.memory` - - :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_store_resources.requests.memory` - - Note that the value must be a decimal unit! - Please also note that - if no explicit memory limit is configured - the helm chart default is used which is not optimal. - You should configure both memory limit and request - which are recommended to have the same value. - - Persistence - *********** - - With `release/v3.0 · Tarook · GitLab `__, - persistence for Thanos components has been reworked. - By default, Thanos components use emptyDirs. - Depending on the size of the cluster and the metrics - flying around, Thanos components may need more disk - than the host node can provide them and in that cases - it makes sense to configure persistence. - - If you want to enable persistence for Thanos components, - you can do so by configuring a storage class - to use and you can specify the persistent volume - size for each component with the following config options: - - - :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_storage_class` - - :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_storegateway_size` - - :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_compactor_size` - - .. _cluster-configuration.prometheus-configuration.updating-immutable-options: - - Updating immutable options - ************************** - - Some options are immutable when deployed. - If you want to change them nonetheless, follow these manual steps: - 1. Increase the size of the corresponding PVC - 2. Delete the stateful set: ``kubectl delete -n monitoring sts --cascade=false thanos-`` - 3. Re-deploy it with the LCM: ``AFLAGS="--diff --tags thanos" bash managed-k8s/actions/apply-k8s-supplements.sh`` - ''; + _docs.preface = builtins.readFile ./preface.rst; install = mkOption { description = '' @@ -252,17 +156,17 @@ in { }; } ); + apply = builtins.map ( + v: + v + // lib.optionalAttrs (v ? write_relabel_configs) (builtins.trace + "WARNING: yk8s.k8s-service-layer.prometheus.remote_writes.[].write_label_configs is deprecated. Please use writeRelabelConfigs instead." + { + writeRelabelConfigs = v.write_relabel_configs; + }) + ); }; - grafana_admin_secret_name = mkOption { - type = k8sSecretName; - default = "cah-grafana-admin"; - }; - - grafana_dashboard_enable_multicluster_support = mkEnableOption '' - referencing multiple K8s clusters by a single Grafana datasource. - ''; - nvidia_dcgm_exporter_helm_repo_url = mkOption { type = helmChartRepoUrl; default = "https://nvidia.github.io/dcgm-exporter/helm-charts"; @@ -303,6 +207,11 @@ in { default = "prometheus-operated"; }; + grafana_admin_secret_name = mkOption { + type = types.str; + default = "cah-grafana-admin"; + }; + prometheus_persistent_storage_class = mkOption { description = '' Configure persistent storage for Prometheus @@ -322,84 +231,6 @@ in { default = "50Gi"; }; - use_grafana = mkOption { - description = "Enable grafana"; - type = types.bool; - default = true; - }; - - grafana_root_url = mkOption { - description = '' - The full public facing url you use in browser, used for redirects and emails - ''; - type = with types; nullOr httpxHostPathUrl; - default = null; - }; - - grafana_persistent_storage_class = mkOption { - description = '' - If this variable is defined, Grafana will store its data in a PersistentVolume - in the defined StorageClass. Otherwise, persistence is disabled for Grafana. - The value has to be a valid StorageClass available in your cluster. - ''; - type = with types; nullOr k8sStorageClassName; - default = null; - }; - - use_thanos = mkEnableOption "use of Thanos"; - - manage_thanos_bucket = mkOption { - description = '' - Let terraform create an object storage container / bucket for you if `true`. - If set to `false` one must provide a valid configuration via Vault. - See :ref:`thanos.custom-bucket-management` for details. - ''; - type = types.bool; - default = true; - }; - - thanos_chart_version = mkHelmChartVersionOption { - # renovate: datasource=helm depName=thanos registryUrl=https://charts.bitnami.com/bitnami - default = "17.2.3"; - }; - - thanos_storage_class = mkOption { - description = '' - Thanos uses emptyDirs by default for its components - for faster access. - If that's not feasible, a storage class can be set to - enable persistence and the size for each component volume - can be configured. - Note that switching between persistence requires - manual intervention and it may be necessary to reinstall - the helm chart completely. - ''; - type = with types; nullOr k8sStorageClassName; - default = null; - }; - - thanos_storegateway_size = mkOption { - description = '' - You can explicitly set the PV size for each component. - If left undefined, the helm chart defaults will be used - - Immutable when deployed. (See also :ref:`cluster-configuration.prometheus-configuration.updating-immutable-options`) - ''; - type = with types; nullOr k8sQuantity; - default = null; - }; - - thanos_compactor_size = mkOption { - description = '' - You can explicitly set the PV size for each component. - If left undefined, the helm chart defaults will be used - - Immutable when deployed. (See also :ref:`cluster-configuration.prometheus-configuration.updating-immutable-options`) - ''; - type = with types; nullOr k8sQuantity; - default = null; - }; - alertmanager_replicas = mkOption { description = '' How many replicas of the alertmanager should be deployed inside the cluster @@ -420,43 +251,10 @@ in { default = null; example = lib.options.literalExpression "\"\${scheduling_key_prefix}/monitoring\""; }; - thanos_store_in_memory_max_size = mkOption { - description = '' - https://thanos.io/tip/components/store.md/#in-memory-index-cache - Note: Unit must be specified as decimal! (MB,GB) - This value should be chosen in a sane matter based on - thanos_store_memory_request and thanos_store_memory_limit - ''; - type = with types; nullOr bytesPower10; - default = null; - }; - thanos_objectstorage_container_name = mkOption { - type = openstackSwiftContainerName; - default = "${config.yk8s.infra.cluster_name}-monitoring-thanos-data"; - defaultText = "\${config.yk8s.infra.cluster_name}-monitoring-thanos-data"; - }; - thanos_objectstorage_config_file = mkOption { - description = '' - Note: The given path is interpreted as being relative to the cluster repo's config directory. - ''; - # NOTE: Not using `pathInStore` here because the expected file contains secrets - # TODO: Eliminate config option and store secrets solely in Vault - type = with types; nullOr relativePosixPath; - default = null; - example = "./monitoring/thanos_objectstorage.config"; - }; internet_probe = mkEnableOption '' scraping external targets via blackbox exporter https://github.com/prometheus-community/helm-charts/tree/main/charts/prometheus-blackbox-exporter ''; - thanos_query_additional_store_endpoints = mkOption { - description = '' - Provide a list of DNS endpoints for additional thanos store endpoints. - The endpoint will be extended to `dnssrv+_grpc._tcp.{{ endpoint }}.monitoring.svc.cluster.local`. - ''; - type = with types; listOf subdomainLabel; - default = []; - }; blackbox_version = mkHelmChartVersionOption { # renovate: datasource=helm depName=prometheus-blackbox-exporter registryUrl=https://prometheus-community.github.io/helm-charts default = "11.3.1"; @@ -555,6 +353,7 @@ in { }; }); }; + common_labels = mkOption { description = '' If at least one common_label is defined, Prometheus will be created with selectors diff --git a/nix/yk8s/k8s-supplements/monitoring/grafana.nix b/nix/yk8s/k8s-supplements/monitoring/grafana.nix new file mode 100644 index 0000000000000000000000000000000000000000..f30244bb9f55030119181e239f54cb61b3d5d9f3 --- /dev/null +++ b/nix/yk8s/k8s-supplements/monitoring/grafana.nix @@ -0,0 +1,76 @@ +{ + config, + lib, + yk8s-lib, + ... +}: let + cfg = config.yk8s.k8s-service-layer.prometheus; + modules-lib = import ../../lib/modules.nix {inherit lib;}; + inherit (modules-lib) mkRemovedOptionModule mkRenamedResourceOptionModule mkMultiResourceOptionsModule; + inherit (lib) mkEnableOption mkOption types; + inherit + (yk8s-lib.types) + httpxHostPathUrl + k8sSecretName + k8sStorageClassName + ; +in { + imports = [ + (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "grafana_plugins"] "") + + (mkMultiResourceOptionsModule ["k8s-service-layer" "prometheus"] { + description = '' + GRAFANA POD RESOURCE LIMITS + The following limits are applied to the respective pods. + Note that the Prometheus limits are chosen fairly conservatively and may need + tuning for larger and smaller clusters. + By default, we prefer to set limits in such a way that the Pods end up in the + Guaranteed QoS class (i.e. both CPU and Memory limits and requests set to the + same value). + ''; + resources = { + grafana.memory.limit = "512Mi"; + grafana.cpu.request = "100m"; + grafana.cpu.example = "500m"; + }; + }) + (mkRenamedResourceOptionModule ["k8s-service-layer" "prometheus"] [ + "grafana" + ]) + ]; + + options.yk8s.k8s-service-layer.prometheus = { + grafana_admin_secret_name = mkOption { + type = k8sSecretName; + default = "cah-grafana-admin"; + }; + + grafana_dashboard_enable_multicluster_support = mkEnableOption '' + referencing multiple K8s clusters by a single Grafana datasource. + ''; + + use_grafana = mkOption { + description = "Enable grafana"; + type = types.bool; + default = true; + }; + + grafana_root_url = mkOption { + description = '' + The full public facing url you use in browser, used for redirects and emails + ''; + type = with types; nullOr httpxHostPathUrl; + default = null; + }; + + grafana_persistent_storage_class = mkOption { + description = '' + If this variable is defined, Grafana will store its data in a PersistentVolume + in the defined StorageClass. Otherwise, persistence is disabled for Grafana. + The value has to be a valid StorageClass available in your cluster. + ''; + type = with types; nullOr k8sStorageClassName; + default = null; + }; + }; +} diff --git a/nix/yk8s/k8s-supplements/monitoring/helm_prometheus_stack/default.nix b/nix/yk8s/k8s-supplements/monitoring/helm_prometheus_stack/default.nix new file mode 100644 index 0000000000000000000000000000000000000000..1ddd7d1d603f6f3e7e7a582f0fc6f67b09b29ab9 --- /dev/null +++ b/nix/yk8s/k8s-supplements/monitoring/helm_prometheus_stack/default.nix @@ -0,0 +1,343 @@ +{ + config, + lib, + yk8s-lib, + ... +}: let + cfg = config.yk8s.k8s-service-layer.prometheus; + modules-lib = import ../../../lib/modules.nix {inherit lib;}; + inherit (modules-lib) mkHelmValuesModule; + inherit (yk8s-lib) mkAffinity mkTolerations; +in { + imports = [ + (mkHelmValuesModule "k8s-service-layer.prometheus" "prometheus_stack") + + ./grafana.nix + ./prometheus.nix + ]; + config.yk8s.k8s-service-layer.prometheus.prometheus_stack_default_values = let + affinity = mkAffinity {inherit (cfg) scheduling_key;}; + tolerations = mkTolerations cfg.scheduling_key; + in { + commonLabels = cfg.common_labels; + priorityClassName = "system-cluster-critical"; + defaultRules = { + create = true; + rules = { + etcd = false; # disabled for now + kubeApiserver = false; # https://github.com/prometheus-community/helm-charts/issues/1283 + }; + }; + + global = { + rbac = { + create = true; + }; + }; + alertmanager = { + enabled = true; + alertmanagerSpec = + { + priorityClassName = "system-cluster-critical"; + replicas = cfg.alertmanager_replicas; + } + // lib.optionalAttrs cfg.allow_external_rules { + ConfigMatcherStrategy = { + type = "None"; + }; + }; + serviceMonitor = { + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + inherit affinity tolerations; + }; + + kubeApiServer = { + enabled = true; + serviceMonitor = { + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_namespace" + "__meta_kubernetes_service_name" + "__meta_kubernetes_endpoint_port_name" + ]; + action = "keep"; + regex = "default;kubernetes;https"; + } + { + targetLabel = "__address__"; + replacement = "kubernetes.default.svc:443"; + } + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + }; + kubelet = { + enabled = true; + serviceMonitor = { + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + { + sourceLabels = [ + "__metrics_path__" + ]; + targetLabel = "metrics_path"; + action = "replace"; + } + ]; + }; + }; + kubeControllerManager = { + enabled = true; + service = { + port = 10257; + targetPort = 10257; + }; + serviceMonitor = { + enabled = true; + https = true; + insecureSkipVerify = true; + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + }; + coreDNS = { + enabled = true; + serviceMonitor = { + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + }; + kubeEtcd = { + enabled = true; + service = { + enabled = true; + port = 2381; + targetPort = 12381; + selector = { + "app.kubernetes.io/name" = "etcd-proxy-metrics"; + }; + ipDualStack = let + inherit (config.yk8s.infra) ipv4_enabled ipv6_enabled; + in { + enabled = ipv4_enabled && ipv6_enabled; + ipFamilies = + (lib.optional ipv4_enabled "IPv4") + ++ (lib.optional ipv6_enabled "IPv6"); + ipFamilyPolicy = + if ipv4_enabled && ipv6_enabled + then "PreferDualStack" + else "SingleStack"; + }; + }; + serviceMonitor = { + enabled = true; + scheme = "https"; + insecureSkipVerify = false; + caFile = "/etc/prometheus/secrets/etcd-metrics-proxy/server.crt"; + certFile = "/etc/prometheus/secrets/etcd-metrics-proxy/client.crt"; + keyFile = "/etc/prometheus/secrets/etcd-metrics-proxy/client.key"; + }; + }; + kubeScheduler = { + enabled = true; + service = { + enabled = true; + port = 10259; + targetPort = 10259; + }; + serviceMonitor = { + enabled = true; + https = true; + insecureSkipVerify = true; + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + }; + kubeProxy = { + enabled = true; + serviceMonitor = { + enabled = true; + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + }; + kubeStateMetrics = { + enabled = true; + serviceMonitor = { + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + }; + kube-state-metrics = { + priorityClassName = "system-cluster-critical"; + rbac = { + create = true; + pspEnabled = false; + }; + customLabels = cfg.common_labels; + metricLabelsAllowlist = [ + "namespaces=[*]" + ]; + }; + nodeExporter = { + enabled = true; + ## Use the value configured in prometheus-node-exporter.podLabels + jobLabel = "jobLabel"; + }; + ## Configuration for prometheus-node-exporter subchart + ## + prometheus-node-exporter = { + priorityClassName = "system-node-critical"; + namespaceOverride = ""; + podLabels = { + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + jobLabel = "node-exporter"; + }; + extraArgs = [ + "--collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+|var/lib/kubelet/.+)($|/)" + "--collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$" + ]; + prometheus = { + monitor = { + enabled = true; + additionalLabels = cfg.common_labels; + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + { + sourceLabels = [ + "__meta_kubernetes_node_labelpresent_node_role_kubernetes_io_control_plane" + ]; + targetLabel = "role"; + regex = "^true$"; + replacement = "control-plane"; + } + { + sourceLabels = [ + "__meta_kubernetes_node_labelpresent_node_role_kubernetes_io_worker" + ]; + targetLabel = "role"; + regex = "^true$"; + replacement = "worker"; + } + ]; + attachMetadata.node = true; + }; + }; + }; + prometheusOperator = { + enabled = true; + priorityClassName = "system-cluster-critical"; + admissionWebhooks = { + patch = { + priorityClassName = "system-cluster-critical"; + }; + }; + resources = cfg.operator_resources; + serviceMonitor = { + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + inherit affinity tolerations; + }; + }; +} diff --git a/nix/yk8s/k8s-supplements/monitoring/helm_prometheus_stack/grafana.nix b/nix/yk8s/k8s-supplements/monitoring/helm_prometheus_stack/grafana.nix new file mode 100644 index 0000000000000000000000000000000000000000..eae6da1da365fe60cb1c338f883fbc803af8c23d --- /dev/null +++ b/nix/yk8s/k8s-supplements/monitoring/helm_prometheus_stack/grafana.nix @@ -0,0 +1,146 @@ +{ + config, + lib, + yk8s-lib, + ... +}: let + cfg = config.yk8s.k8s-service-layer.prometheus; + inherit (yk8s-lib) mkAffinity mkTolerations; +in { + config.yk8s.k8s-service-layer.prometheus.prometheus_stack_default_values = { + grafana = { + priorityClassName = "system-cluster-critical"; + enabled = cfg.use_grafana; + persistence = { + enabled = cfg.grafana_persistent_storage_class != null; + storageClassName = cfg.grafana_persistent_storage_class; + }; + admin = { + existingSecret = cfg.grafana_admin_secret_name; + userKey = "admin-user"; + passwordKey = "admin-password"; + }; + resources = cfg.grafana_resources; + tolerations = mkTolerations cfg.scheduling_key; + affinity = + mkAffinity { + inherit (cfg) scheduling_key; + } + // lib.optionalAttrs (cfg.grafana_persistent_storage_class != null) { + pod_affinity_key = "app.kubernetes.io/name"; + pod_affinity_operator = "In"; + pod_affinity_values = ["grafana"]; + }; + datasources = { + "datasources.yaml" = { + apiVersion = 1; + datasources = + lib.optional cfg.use_thanos + { + name = "thanos"; + type = "prometheus"; + access = "proxy"; + orgId = 1; + url = "http://thanos-query.{{ monitoring_namespace }}.svc:9090"; + version = 1; + editable = false; + }; + }; + }; + sidecar = { + dashboards = + { + enabled = true; + label = "grafana_dashboard"; + searchNamespace = "ALL"; + folderAnnotation = "customer-dashboards"; + provider = { + foldersFromFilesStructure = true; + }; + } + // lib.optionalAttrs cfg.grafana_dashboard_enable_multicluster_support { + multicluster.global.enabled = true; + }; + }; + serviceMonitor = { + enabled = true; + labels = cfg.common_labels; + # # https://github.com/prometheus-community/helm-charts/issues/1776 + interval = "30s"; + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + dashboards = { + managed = + { + dashboard-calico = { + gnetId = 12175; + revision = 5; + datasource = "Prometheus"; + }; + } + // (lib.optionalAttrs config.yk8s.k8s-service-layer.rook.enabled { + dashboard-ceph-cluster = { + gnetId = 2842; + revision = 14; + datasource = "Prometheus"; + }; + dashboard-ceph-osd-single = { + gnetId = 5336; + revision = 5; + datasource = "Prometheus"; + }; + dashboard-ceph-pools = { + gnetId = 5342; + revision = 5; + datasource = "Prometheus"; + }; + }) + // (lib.optionalAttrs cfg.use_thanos { + dashboard-thanos = { + gnetId = 12937; + revision = 4; + datasource = "Prometheus"; + }; + }) + // (lib.optionalAttrs config.yk8s.k8s-service-layer.ingress.enabled { + dashboard-nginx-ingress = { + gnetId = 9614; + revision = 1; + datasource = "Prometheus"; + }; + }); + }; + + dashboardProviders = { + "managed-dashboard-provider.yaml" = { + apiVersion = 1; + providers = [ + { + name = "managed-dashboards"; + folder = "managed-dashboards"; + options = { + path = "/var/lib/grafana/dashboards/managed"; + }; + } + ]; + }; + }; + "grafana.ini" = { + server = lib.optionalAttrs (cfg.grafana_root_url != null) { + root_url = cfg.grafana_root_url; + }; + }; + }; + }; +} diff --git a/nix/yk8s/k8s-supplements/monitoring/helm_prometheus_stack/prometheus.nix b/nix/yk8s/k8s-supplements/monitoring/helm_prometheus_stack/prometheus.nix new file mode 100644 index 0000000000000000000000000000000000000000..64a966852f95e2bcbd6ccd209eb0392163b76af6 --- /dev/null +++ b/nix/yk8s/k8s-supplements/monitoring/helm_prometheus_stack/prometheus.nix @@ -0,0 +1,111 @@ +{ + config, + lib, + yk8s-lib, + ... +}: let + cfg = config.yk8s.k8s-service-layer.prometheus; + inherit (yk8s-lib) mkAffinity mkTolerations; +in { + config.yk8s.k8s-service-layer.prometheus.prometheus_stack_default_values = { + prometheus = { + enabled = true; + thanosService = { + enabled = "{{ monitoring_use_thanos | bool }}"; + }; + thanosServiceMonitor = { + enabled = "{{ monitoring_use_thanos | bool }}"; + }; + serviceMonitor = { + relabelings = [ + { + sourceLabels = [ + "__meta_kubernetes_pod_node_name" + ]; + separator = ";"; + regex = "^(.*)$"; + targetLabel = "nodename"; + replacement = "$1"; + action = "replace"; + } + ]; + }; + prometheusSpec = + { + priorityClassName = "system-cluster-critical"; + remoteWrite = cfg.remote_writes; ## TODO : + # {% if remote_write.basic_auth_secret_name is not none %} + # basicAuth: + # username: + # name: {{ remote_write.basic_auth_secret_name | to_json }} + # key: username + # password: + # name: {{ remote_write.basic_auth_secret_name | to_json }} + # key: password + # {% endif %} + secrets = [ + "etcd-metrics-proxy" + ]; + serviceMonitorSelectorNilUsesHelmValues = cfg.common_labels != {}; + + containers = [ + { + name = "prometheus"; + readinessProbe = { + failureThreshold = 1000; + }; + } + ]; + resources = "cfg.prometheus_resources"; + affinity = mkAffinity {inherit (cfg) scheduling_key;}; + tolerations = mkTolerations cfg.scheduling_key; + } + // lib.optionalAttrs (cfg.prometheus_persistent_storage_class != null) { + storageSpec = { + volumeClaimTemplate = { + spec = { + storageClassName = cfg.prometheus_persistent_storage_class != null; + accessModes = [ + "ReadWriteOnce" + ]; + resources = { + requests = { + storage = cfg.prometheus_persistent_storage_resource_request; + }; + }; + }; + }; + }; + } + // lib.optionalAttrs (cfg.common_labels != {}) { + serviceMonitorSelector = { + matchLabels = cfg.common_labels; + }; + } + // lib.optionalAttrs (cfg.use_thanos) { + thanos = { + objectStorageConfig = + { + optional = false; + } + // ( + let + existingSecret = { + name = "thanos-sidecar-bucket-credentials-config"; + key = "thanos.yaml"; + }; + in + if (lib.toInt (lib.versions.major cfg.prometheus_stack_version)) >= 51 + then {inherit existingSecret;} + else {inherit (existingSecret) name key;} + ); + }; + } + // lib.optionalAttrs cfg.allow_external_rules { + ruleSelectorNilUsesHelmValues = false; + ruleSelector = {}; + ruleNamespaceSelector = {}; + }; + }; + }; +} diff --git a/nix/yk8s/k8s-supplements/monitoring/preface.rst b/nix/yk8s/k8s-supplements/monitoring/preface.rst new file mode 100644 index 0000000000000000000000000000000000000000..0c792af7955e7072b2a72c4b4b6d79aa528c8326 --- /dev/null +++ b/nix/yk8s/k8s-supplements/monitoring/preface.rst @@ -0,0 +1,65 @@ +The used prometheus-based monitoring setup will be explained in more +detail soon :) + +.. note:: + +To enable prometheus, +:ref:`configuration-options.yk8s.k8s-service-layer.prometheus.install` +and +:ref:`configuration-options.yk8s.kubernetes.monitoring.enabled` +need to be set to ``true``. + + +Tweak Thanos Configuration +"""""""""""""""""""""""""" + +index-cache-size / in-memory-max-size +************************************* + +Thanos is unaware of its Kubernetes limits +which can lead to OOM kills of the storegateway +if a lot of metrics are requested. + +This can be prevented by tuning the following config options: + +- :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_store_in_memory_max_size` +- :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_store_resources.limits.memory` +- :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_store_resources.requests.memory` + +Note that the value must be a decimal unit! +Please also note that +if no explicit memory limit is configured +the helm chart default is used which is not optimal. +You should configure both memory limit and request +which are recommended to have the same value. + +Persistence +*********** + +With `release/v3.0 · Tarook · GitLab `__, +persistence for Thanos components has been reworked. +By default, Thanos components use emptyDirs. +Depending on the size of the cluster and the metrics +flying around, Thanos components may need more disk +than the host node can provide them and in that cases +it makes sense to configure persistence. + +If you want to enable persistence for Thanos components, +you can do so by configuring a storage class +to use and you can specify the persistent volume +size for each component with the following config options: + +- :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_storage_class` +- :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_storegateway_size` +- :ref:`configuration-options.yk8s.k8s-service-layer.prometheus.thanos_compactor_size` + +.. _cluster-configuration.prometheus-configuration.updating-immutable-options: + +Updating immutable options +************************** + +Some options are immutable when deployed. +If you want to change them nonetheless, follow these manual steps: +1. Increase the size of the corresponding PVC +2. Delete the stateful set: ``kubectl delete -n monitoring sts --cascade=false thanos-`` +3. Re-deploy it with the LCM: ``AFLAGS="--diff --tags thanos" bash managed-k8s/actions/apply-k8s-supplements.sh`` diff --git a/nix/yk8s/k8s-supplements/monitoring/thanos.nix b/nix/yk8s/k8s-supplements/monitoring/thanos.nix new file mode 100644 index 0000000000000000000000000000000000000000..2993889c9e4311817fc3ba8b921e89f6f7db0995 --- /dev/null +++ b/nix/yk8s/k8s-supplements/monitoring/thanos.nix @@ -0,0 +1,238 @@ +{ + config, + lib, + yk8s-lib, + ... +}: let + cfg = config.yk8s.k8s-service-layer.prometheus; + modules-lib = import ../../lib/modules.nix {inherit lib;}; + inherit (modules-lib) mkRemovedOptionModule mkRenamedResourceOptionModule mkMultiResourceOptionsModule; + inherit (lib) mkEnableOption mkOption types; + inherit (yk8s-lib.options) mkHelmReleaseOptions; + inherit (yk8s-lib.k8s) mkAffinity mkTolerations; + inherit + (yk8s-lib.types) + bytesPower10 + k8sQuantity + k8sStorageClassName + openstackSwiftContainerName + relativePosixPath + subdomainLabel + ; +in { + imports = [ + (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "use_helm_thanos"] "") + (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "thanos_metadata_volume_size"] "") + (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "thanos_metadata_volume_storage_class"] "") + (mkRemovedOptionModule ["k8s-service-layer" "prometheus" "thanos_objectstorage_config_path"] "Use `thanos_objectstorage_config_file` instead.") + + (mkMultiResourceOptionsModule ["k8s-service-layer" "prometheus"] { + description = '' + THANOS POD RESOURCE LIMITS + The following limits are applied to the respective pods. + Note that the Prometheus limits are chosen fairly conservatively and may need + tuning for larger and smaller clusters. + By default, we prefer to set limits in such a way that the Pods end up in the + Guaranteed QoS class (i.e. both CPU and Memory limits and requests set to the + same value). + ''; + resources = { + thanos_sidecar.memory.limit = "256Mi"; + thanos_sidecar.cpu.request = "500m"; + + thanos_query.memory.limit = "786Mi"; + thanos_query.cpu.request = "100m"; + thanos_query.cpu.example = "1"; + + thanos_compact.memory.limit = "200Mi"; + thanos_compact.cpu.request = "100m"; + + thanos_store.memory.limit = "2Gi"; + thanos_store.cpu.request = "100m"; + thanos_store.cpu.example = "500m"; + }; + }) + (mkRenamedResourceOptionModule ["k8s-service-layer" "prometheus"] [ + "thanos_sidecar" + "thanos_query" + "thanos_compact" + "thanos_store" + ]) + ]; + + options.yk8s.k8s-service-layer.prometheus = { + use_thanos = mkEnableOption "use of Thanos"; + + manage_thanos_bucket = mkOption { + description = '' + Let terraform create an object storage container / bucket for you if `true`. + If set to `false` one must provide a valid configuration via Vault. + See :ref:`thanos.custom-bucket-management` for details. + ''; + type = types.bool; + default = true; + }; + + thanos_chart_version = mkHelmChartVersionOption { + # renovate: datasource=helm depName=thanos registryUrl=https://charts.bitnami.com/bitnami + default = "17.2.3"; + }; + + thanos_storage_class = mkOption { + description = '' + Thanos uses emptyDirs by default for its components + for faster access. + If that's not feasible, a storage class can be set to + enable persistence and the size for each component volume + can be configured. + Note that switching between persistence requires + manual intervention and it may be necessary to reinstall + the helm chart completely. + ''; + type = with types; nullOr k8sStorageClassName; + default = null; + }; + + thanos_storegateway_size = mkOption { + description = '' + You can explicitly set the PV size for each component. + If left undefined, the helm chart defaults will be used + + Immutable when deployed. (See also :ref:`cluster-configuration.prometheus-configuration.updating-immutable-options`) + ''; + type = with types; nullOr k8sQuantity; + default = null; + }; + + thanos_compactor_size = mkOption { + description = '' + You can explicitly set the PV size for each component. + If left undefined, the helm chart defaults will be used + + Immutable when deployed. (See also :ref:`cluster-configuration.prometheus-configuration.updating-immutable-options`) + ''; + type = with types; nullOr k8sQuantity; + default = null; + }; + thanos_store_in_memory_max_size = mkOption { + description = '' + https://thanos.io/tip/components/store.md/#in-memory-index-cache + Note: Unit must be specified as decimal! (MB,GB) + This value should be chosen in a sane matter based on + thanos_store_memory_request and thanos_store_memory_limit + ''; + type = with types; nullOr bytesPower10; + default = null; + }; + thanos_objectstorage_container_name = mkOption { + type = openstackSwiftContainerName; + default = "${config.yk8s.infra.cluster_name}-monitoring-thanos-data"; + defaultText = "\${config.yk8s.infra.cluster_name}-monitoring-thanos-data"; + }; + thanos_objectstorage_config_file = mkOption { + description = '' + Note: The given path is interpreted as being relative to the cluster repo's config directory. + ''; + # NOTE: Not using `pathInStore` here because the expected file contains secrets + # TODO: Eliminate config option and store secrets solely in Vault + type = with types; nullOr relativePosixPath; + default = null; + example = "./monitoring/thanos_objectstorage.config"; + }; + thanos_query_additional_store_endpoints = mkOption { + description = '' + Provide a list of DNS endpoints for additional thanos store endpoints. + The endpoint will be extended to `dnssrv+_grpc._tcp.{{ endpoint }}.monitoring.svc.cluster.local`. + ''; + type = with types; listOf subdomainLabel; + default = []; + }; + helm = mkHelmReleaseOptions { + # TODO adapt from ingress and add renames + descriptionName = "ingress-nginx"; + defaultRepoUrl = "https://kubernetes.github.io/ingress-nginx"; + defaultChartRef = "ingress-nginx"; + # renovate: datasource=helm depName=ingress-nginx registryUrl=https://kubernetes.github.io/ingress-nginx + defaultChartVersion = "4.13.2"; + defaultReleaseNamespace = "k8s-svc-ingress"; + defaultReleaseName = "ingress"; + valuesDocUrl = "https://github.com/kubernetes/ingress-nginx/blob/main/charts/ingress-nginx/values.yaml"; + chartOptions = {}; + }; + }; + + config.yk8s.k8s-service-layer.prometheus.thanos.helm.values = let + affinity = mkAffinity {inherit (cfg) scheduling_key;}; + tolerations = mkTolerations cfg.scheduling_key; + in { + global = lib.optionalAttrs (cfg.thanos_storage_class != null) { + storageClass = cfg.thanos_storage_class; + # Original containers have been substituted, see https://github.com/bitnami/containers/issues/83267 + security.allowInsecureImages = true; + }; + image.repository = "bitnamilegacy/thanos"; + existingObjstoreSecret = cfg.thanos_config_secret_name; + compactor = { + enabled = true; + extraFlags = [ + "--no-debug.halt-on-error" # must exit for Kubernetes to restart it + ]; + retentionResolutionRaw = cfg.thanos_retention_resolution_raw; + retentionResolution5m = cfg.thanos_retention_resolution_5m; + retentionResolution1h = cfg.thanos_retention_resolution_1h; + resources = cfg.thanos_compact_resources; + inherit affinity tolerations; + persistence = + { + enabled = cfg.thanos_storage_class != null; + } + // lib.optionalAttrs (cfg.thanos_compactor_size != null) { + size = cfg.thanos_compactor_size; + }; + }; + + storegateway = + { + enabled = true; + extraFlags = + lib.optional (cfg.thanos_store_in_memory_max_size != null) + "--index-cache-size=${cfg.thanos_store_in_memory_max_size}"; + resources = cfg.thanos_store_resources; + inherit affinity tolerations; + pdb.create = false; + replicaCount = 1; + persistence.enabled = cfg.thanos_storage_class != null; + } + // lib.optionalAttrs (cfg.thanos_storegateway_size != null) { + size = cfg.thanos_storegateway_size; + }; + + query = { + enabled = true; + resources = cfg.thanos_query_resources; + dnsDiscovery = { + enabled = true; + sidecarsService = "prometheus-operated"; + sidecarsNamespace = cfg.namespace; + }; + inherit affinity tolerations; + pdb.create = false; + replicaCount = 1; + extraFlags = [ + "--query.auto-downsampling" + "--query.timeout=1m" + ]; + }; + queryFrontend = { + enabled = false; + }; + + metrics = { + enabled = true; + serviceMonitor = { + enabled = true; + labels = cfg.common_labels; + }; + }; + }; +} diff --git a/nix/yk8s/lib/k8s.nix b/nix/yk8s/lib/k8s.nix index 2edb66be40518c459fd1332c0a9f2f195273a712..bcf16b9a4204e0499ec802fe92359d6133b42b3a 100644 --- a/nix/yk8s/lib/k8s.nix +++ b/nix/yk8s/lib/k8s.nix @@ -13,7 +13,7 @@ } ]; - mkAffinities = { + mkAffinity = { scheduling_key ? null, pod_affinity_key ? null, pod_affinity_operator ? "Exists",