diff --git a/docs/_releasenotes/1250.feature.2.labels-taints b/docs/_releasenotes/1250.feature.2.labels-taints new file mode 100644 index 0000000000000000000000000000000000000000..76a80c3d539ee5b1fa39ebf67cddd378ae60f331 --- /dev/null +++ b/docs/_releasenotes/1250.feature.2.labels-taints @@ -0,0 +1 @@ +Labels and taints are now fully configurable, see :ref:`configuration-options.yk8s.node-scheduling` diff --git a/docs/user/reference/options/yk8s.node-scheduling.rst b/docs/user/reference/options/yk8s.node-scheduling.rst index 2144907b97a977a99a35f91af7ac6845a51bb01f..becf57babd052fc4b5eb113bf1b4361d215c487e 100644 --- a/docs/user/reference/options/yk8s.node-scheduling.rst +++ b/docs/user/reference/options/yk8s.node-scheduling.rst @@ -20,10 +20,12 @@ More details about the labels and taints configuration can be found Labels are assigned to a node during LCM rollout only! +Can be set to ``null`` in order to remove the label. + **Type:**:: - attribute set of list of Kubernetes label string + attribute set of ((list of Kubernetes label string) or attribute set of (null or Kubernetes label value)) **Default:**:: @@ -34,24 +36,18 @@ Labels are assigned to a node during LCM rollout only! **Example:**:: { - managed-k8s-worker-0 = [ - "${scheduling_key_prefix}/storage=true" - ]; + managed-k8s-worker-0 = { + "example.org/monitoring" = "true"; + "example.org/storage" = "true"; + }; managed-k8s-worker-1 = [ - "${scheduling_key_prefix}/monitoring=true" - ]; - managed-k8s-worker-2 = [ - "${scheduling_key_prefix}/storage=true" - ]; - managed-k8s-worker-3 = [ - "${scheduling_key_prefix}/monitoring=true" - ]; - managed-k8s-worker-4 = [ - "${scheduling_key_prefix}/storage=true" - ]; - managed-k8s-worker-5 = [ - "${scheduling_key_prefix}/monitoring=true" + "example.org/storage=true" + "example.org/monitoring=true" ]; + managed-k8s-worker-2 = { + "example.org/monitoring" = null; + "example.org/storage" = null; + }; } @@ -94,10 +90,12 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/node-schedul Taints are assigned to a node during LCM rollout only! +Can be set to ``null`` in order to remove the taint. + **Type:**:: - attribute set of list of Kubernetes taint string + attribute set of ((list of Kubernetes taint string) or attribute set of (null or (submodule))) **Default:**:: @@ -108,15 +106,18 @@ Taints are assigned to a node during LCM rollout only! **Example:**:: { - managed-k8s-worker-0 = [ - "${scheduling_key_prefix}/storage=true:NoSchedule" - ]; + managed-k8s-worker-0 = { + "examply.org/storage" = { + effect = "NoSchedule"; + value = "true"; + }; + }; managed-k8s-worker-2 = [ - "${scheduling_key_prefix}/storage=true:NoSchedule" - ]; - managed-k8s-worker-4 = [ - "${scheduling_key_prefix}/storage=true:NoSchedule" + "example.org/storage=true:NoSchedule" ]; + managed-k8s-worker-4 = { + "examply.org/storage" = null; + }; } diff --git a/k8s-core/ansible/roles/k8s-master/tasks/main.yaml b/k8s-core/ansible/roles/k8s-master/tasks/main.yaml index 5ba97325d6ee55222b2b4b53cd0ce991dc5c1e57..5428d553b7bd3609cbf095f5bd53ca248684adaa 100644 --- a/k8s-core/ansible/roles/k8s-master/tasks/main.yaml +++ b/k8s-core/ansible/roles/k8s-master/tasks/main.yaml @@ -313,21 +313,8 @@ # See docs ref:cluster.node-labeling - name: "Ensure K8s node labels" delegate_to: "{{ groups['orchestrator'] | first }}" - vars: - # Dict of configured + managed node labels to apply after join - # managed labels take precedence - node_labels: > - {{ - ( - dict( - k8s_node_labels[inventory_hostname] | default([]) - | map('split', '=', 1) - ) - | combine(_managed_node_labels['masters']) - ) - }} when: - - node_labels | length > 0 + - k8s_node_labels[inventory_hostname] | length > 0 kubernetes.core.k8s: state: patched api_version: v1 @@ -335,28 +322,23 @@ name: "{{ inventory_hostname }}" definition: metadata: - labels: "{{ node_labels }}" + labels: "{{ k8s_node_labels[inventory_hostname] }}" -- name: "Ensure K8s node taints" +- name: "Ensure present K8s node taints" delegate_to: "{{ groups['orchestrator'] | first }}" - vars: - # List of configured node taints (as dicts) to apply after join - # (taint strings parsed to dict, - # empty values are omitted, effect defaults to NoExecute) - node_taints: > - {{ - ( - k8s_node_taints[inventory_hostname] | default([]) - | map('regex_replace', '^([^:=]+)([=]([^:]+))?([:](.+))?', '{"key": "\1", "value": "\3", "effect": "\5"}') - | map('regex_replace', ', "value": ""', '') - | map('regex_replace', '"effect": ""', '"effect": "NoExecute"') - | map('from_json') - ) - }} when: - - node_taints | length > 0 + - k8s_node_taints_present[inventory_hostname] | length > 0 kubernetes.core.k8s_taint: state: present name: "{{ inventory_hostname }}" - taints: "{{ node_taints }}" + taints: "{{ k8s_node_taints_present[inventory_hostname] }}" + +- name: "Ensure absent K8s node taints" + delegate_to: "{{ groups['orchestrator'] | first }}" + when: + - k8s_node_taints_absent[inventory_hostname] | length > 0 + kubernetes.core.k8s_taint: + state: absent + name: "{{ inventory_hostname }}" + taints: "{{ k8s_node_taints_absent[inventory_hostname] }}" ... diff --git a/k8s-core/ansible/roles/k8s-worker/tasks/main.yaml b/k8s-core/ansible/roles/k8s-worker/tasks/main.yaml index e001050bbed1a2ad7d80a33aabac844bd53f10ff..c5bc8eb5761a2562a29759db9ef07a2dac8f942f 100644 --- a/k8s-core/ansible/roles/k8s-worker/tasks/main.yaml +++ b/k8s-core/ansible/roles/k8s-worker/tasks/main.yaml @@ -78,20 +78,14 @@ - name: "Ensure managed K8s node labels" delegate_to: "{{ groups['orchestrator'] | first }}" vars: - # Dict of configured + gpu related + managed node labels to apply after join - # managed labels take precedence node_labels: > {{ ( - dict( - k8s_node_labels[inventory_hostname] | default([]) - | map('split', '=', 1) - ) + k8s_node_labels[inventory_hostname] | combine( (k8s_is_gpu_cluster and not k8s_virtualize_gpu and ansible_local['gpu-node']['node_has_gpu'] | bool) | ternary({"k8s.yaook.cloud/gpu-node": "true"}, {}) ) - | combine(_managed_node_labels['workers']) ) }} when: node_labels | length > 0 @@ -104,7 +98,7 @@ metadata: labels: "{{ node_labels }}" -- name: "Ensure K8s node taints" +- name: "Ensure present K8s node taints" delegate_to: "{{ groups['orchestrator'] | first }}" vars: # List of configured + gpu related node taints (as dicts) to apply after join @@ -113,11 +107,7 @@ node_taints: > {{ ( - k8s_node_taints[inventory_hostname] | default([]) - | map('regex_replace', '^([^:=]+)([=]([^:]+))?([:](.+))?', '{"key": "\1", "value": "\3", "effect": "\5"}') - | map('regex_replace', ', "value": ""', '') - | map('regex_replace', '"effect": ""', '"effect": "NoExecute"') - | map('from_json') + k8s_node_taints_present[inventory_hostname] ) + ( @@ -131,4 +121,13 @@ state: present name: "{{ inventory_hostname }}" taints: "{{ node_taints }}" + +- name: "Ensure absent K8s node taints" + delegate_to: "{{ groups['orchestrator'] | first }}" + when: + - k8s_node_taints_absent[inventory_hostname] | length > 0 + kubernetes.core.k8s_taint: + state: absent + name: "{{ inventory_hostname }}" + taints: "{{ k8s_node_taints_absent[inventory_hostname] }}" ... diff --git a/k8s-core/ansible/vars/node-labels.yaml b/k8s-core/ansible/vars/node-labels.yaml deleted file mode 100644 index 1e5975791b4c5304adaa011325b6103c8cc7a805..0000000000000000000000000000000000000000 --- a/k8s-core/ansible/vars/node-labels.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -# LCM managed Kubernetes node labels for each Ansible host group -# see docs ref:cluster.node-labeling -# NOTE: Labels set to `null` will be removed if they exist -_managed_node_labels: - masters: - node-role.kubernetes.io/control-plane: "" - node-role.kubernetes.io/worker: null - workers: - node-role.kubernetes.io/worker: "" - node-role.kubernetes.io/control-plane: null - # k8s.yaook.cloud/gpu-node: true # is set conditionally - # TODO: support conditional managed node labels -... diff --git a/nix/yk8s/node-scheduling.nix b/nix/yk8s/node-scheduling.nix index 6037ea55b8f69c641b9119ed19d05062310a587f..34cc561a84a2e7379735141c189b5bcca1807e1a 100644 --- a/nix/yk8s/node-scheduling.nix +++ b/nix/yk8s/node-scheduling.nix @@ -7,11 +7,12 @@ }: let cfg = config.yk8s.node-scheduling; inherit (lib) mkOption types; - inherit (yk8s-lib) mkTopSection mkGroupVarsFile; + inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkInternalOption; inherit (yk8s-lib.types) k8sLabelStr k8sTaintStr + k8sLabelValue ; in { options.yk8s.node-scheduling = mkTopSection { @@ -46,49 +47,114 @@ in { labels = mkOption { description = '' Labels are assigned to a node during LCM rollout only! + + Can be set to ``null`` in order to remove the label. ''; - type = with types; attrsOf (listOf k8sLabelStr); + type = with types; attrsOf (either (listOf k8sLabelStr) (attrsOf (nullOr k8sLabelValue))); default = {}; - example = lib.options.literalExpression '' - { - managed-k8s-worker-0 = [ - "''${scheduling_key_prefix}/storage=true" - ]; - managed-k8s-worker-1 = [ - "''${scheduling_key_prefix}/monitoring=true" - ]; - managed-k8s-worker-2 = [ - "''${scheduling_key_prefix}/storage=true" - ]; - managed-k8s-worker-3 = [ - "''${scheduling_key_prefix}/monitoring=true" - ]; - managed-k8s-worker-4 = [ - "''${scheduling_key_prefix}/storage=true" - ]; - managed-k8s-worker-5 = [ - "''${scheduling_key_prefix}/monitoring=true" - ]; - }''; + example = { + managed-k8s-worker-0."example.org/storage" = "true"; + managed-k8s-worker-0."example.org/monitoring" = "true"; + managed-k8s-worker-1 = ["example.org/storage=true" "example.org/monitoring=true"]; + managed-k8s-worker-2."example.org/storage" = null; + managed-k8s-worker-2."example.org/monitoring" = null; + }; + apply = let + labelListToAttrs = builtins.foldl' (acc: labelStr: let + m = builtins.match "(.*)=(.*)" labelStr; + label = builtins.elemAt m 0; + value = builtins.elemAt m 1; + in + acc + // { + ${label} = value; + }) {}; + in + lib.mapAttrs ( + nodeName: nodeLabels: + if builtins.isAttrs nodeLabels + then nodeLabels + else + labelListToAttrs + nodeLabels + ); }; taints = mkOption { description = '' Taints are assigned to a node during LCM rollout only! + + Can be set to ``null`` in order to remove the taint. ''; - type = with types; attrsOf (listOf k8sTaintStr); + type = with types; + attrsOf (either (listOf k8sTaintStr) (attrsOf (nullOr (submodule { + options = { + value = mkOption { + type = nullOr k8sLabelValue; + default = null; + }; + effect = mkOption { + type = enum ["NoExecute" "NoSchedule" "PreferNoSchedule"]; + default = "NoExecute"; + }; + }; + })))); default = {}; - example = lib.options.literalExpression '' - { - managed-k8s-worker-0 = [ - "''${scheduling_key_prefix}/storage=true:NoSchedule" - ]; - managed-k8s-worker-2 = [ - "''${scheduling_key_prefix}/storage=true:NoSchedule" - ]; - managed-k8s-worker-4 = [ - "''${scheduling_key_prefix}/storage=true:NoSchedule" - ]; - }''; + example = { + managed-k8s-worker-0."examply.org/storage" = { + value = "true"; + effect = "NoSchedule"; + }; + managed-k8s-worker-2 = ["example.org/storage=true:NoSchedule"]; + managed-k8s-worker-4."examply.org/storage" = null; + }; + apply = let + taintListToAttrs = builtins.foldl' (acc: taintStr: let + m = builtins.match "^([^:=]+)([=]([^:]+))?([:](.+))?" taintStr; + taint = builtins.elemAt m 0; + value = builtins.elemAt m 2; + effect = let + e = builtins.elemAt m 4; + in + if e != null + then e + else "NoExecute"; + in + acc + // { + ${taint} = {inherit value effect;}; + }) {}; + in + lib.mapAttrs ( + nodeName: nodeTaints: + if builtins.isAttrs nodeTaints + then nodeTaints + else taintListToAttrs nodeTaints + ); + }; + taints_present = mkInternalOption { + readOnly = true; + type = with types; attrsOf (listOf attrs); + default = + lib.mapAttrs ( + _: nodeTaints: + lib.mapAttrsToList (key: { + value, + effect, + }: {inherit key value effect;}) + (lib.filterAttrs (_: v: v != null) nodeTaints) + ) + cfg.taints; + }; + taints_absent = mkInternalOption { + readOnly = true; + type = with types; attrsOf (listOf attrs); + default = + lib.mapAttrs ( + _: nodeTaints: + lib.mapAttrsToList (key: _: {inherit key;}) + (lib.filterAttrs (_: v: v == null) nodeTaints) + ) + cfg.taints; }; }; config.yk8s.warnings = @@ -100,6 +166,23 @@ in { acc ++ lib.optional (config.yk8s.infra.final_hosts != null && ! builtins.hasAttr e (config.yk8s.infra.final_hosts.all.hosts or {})) "config.yk8s.node-scheduling.taints: taint defined for ${e}, but node not found in config.yk8s.infra.ansible_hosts") [] (builtins.attrNames cfg.taints)); + config.yk8s.node-scheduling.labels = + ( + lib.mapAttrs ( + workerName: _: { + "node-role.kubernetes.io/worker" = ""; + "node-role.kubernetes.io/control-plane" = null; + } + ) + (config.yk8s.infra.ansible_hosts.workers.hosts or {}) + ) + // ( + lib.mapAttrs (masterName: _: { + "node-role.kubernetes.io/control-plane" = ""; + "node-role.kubernetes.io/worker" = null; + }) + (config.yk8s.infra.ansible_hosts.masters.hosts or {}) + ); config.yk8s._inventory_packages = [ (mkGroupVarsFile { inherit cfg; @@ -107,9 +190,10 @@ in { transformations = [ # TODO: remove when deprecated scheduling_key_prefix is dropped (lib.attrsets.filterAttrs (n: _: ! (builtins.elem n ["scheduling_key_prefix"]))) + (lib.attrsets.filterAttrs (name: _: name != "taints")) (lib.attrsets.mapAttrs' (name: value: { name = - if builtins.elem name ["labels" "taints"] + if builtins.elem name ["labels" "taints_present" "taints_absent"] then "k8s_node_${name}" else name; inherit value; @@ -117,7 +201,8 @@ in { ]; unflat = [ ["k8s_node_labels"] - ["k8s_node_taints"] + ["k8s_node_taints_present"] + ["k8s_node_taints_absent"] ]; }) ];