diff --git a/actions/apply-terraform.sh b/actions/apply-terraform.sh index a0fb1a0a1e0d30e231e119c71421994f05f4c0a8..4593527df090acb6674323f8846d9577c37450c2 100755 --- a/actions/apply-terraform.sh +++ b/actions/apply-terraform.sh @@ -39,7 +39,8 @@ elif [ $rc != $RC_NO_DISRUPTION ] && [ $rc != $RC_DISRUPTION ]; then errorf 'error during execution of check_plan.py. Aborting' >&2 exit 4 fi -run terraform -chdir="$terraform_module" apply "$terraform_plan" +run terraform -chdir="$terraform_module" apply "$terraform_plan" | sed '/Outputs:/,$d' # we don't want Terraform to dump all the outputs to the console +terraform -chdir="$terraform_module" output -json > "$terraform_state_dir/outputs.json" if [ "$(jq -r .backend.type "$terraform_state_dir/.terraform/terraform.tfstate")" == 'http' ]; then notef 'Pulling latest Terraform state from Gitlab for disaster recovery purposes.' diff --git a/actions/release-migrations/v11-01-yaml-hosts.sh b/actions/release-migrations/v11-01-yaml-hosts.sh new file mode 100644 index 0000000000000000000000000000000000000000..08946338b6b73a099411f126e1154da8478f582c --- /dev/null +++ b/actions/release-migrations/v11-01-yaml-hosts.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -euo pipefail +actions_dir="$(dirname "$0")/.." + +# shellcheck source=actions/lib.sh +. "$actions_dir/lib.sh" + +notef "Removing obsolete state files..." +run git rm -rf "$terraform_state_dir/rendered" diff --git a/docs/_releasenotes/1250.feature.1.yaml-hosts b/docs/_releasenotes/1250.feature.1.yaml-hosts new file mode 100644 index 0000000000000000000000000000000000000000..ae52d3dc9966ec731300ce4f600c2cf6848358a8 --- /dev/null +++ b/docs/_releasenotes/1250.feature.1.yaml-hosts @@ -0,0 +1 @@ +Ansible hosts can now be defined and referenced via Nix. See :ref:`configuration-options.yk8s.infra.ansible_hosts`. diff --git a/docs/_releasenotes/1250.removal.1.yaml-hosts b/docs/_releasenotes/1250.removal.1.yaml-hosts new file mode 100644 index 0000000000000000000000000000000000000000..6163cbc5cfc5a68d3c1d04155c0f80ad3a91c237 --- /dev/null +++ b/docs/_releasenotes/1250.removal.1.yaml-hosts @@ -0,0 +1 @@ +Importing an existing hosts file via :ref:`configuration-options.yk8s.infra.hosts_file` is deprecated. Hosts can be defined directly via Nix now. The option ``hosts_file`` will be removed at some point in the future. If you want to keep providing your own hosts file after that, convert it to YAML format (see https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html ) and import it like this ``ansible_hosts = yk8s-libs.importYAML ./hosts;``. diff --git a/docs/developer/guide/simulate-bm.rst b/docs/developer/guide/simulate-bm.rst index 090a4fc56c5c3fd82cd51c286298583f5b9b8633..ee665e02e301eb00f81f3e09fc85190f2fd2b3b0 100644 --- a/docs/developer/guide/simulate-bm.rst +++ b/docs/developer/guide/simulate-bm.rst @@ -87,23 +87,17 @@ their ports and associated floating IPs: openstack port delete "$gateway" done -Reconfigure the inventory ``inventory/yaook-k8s/hosts``: +Adapt the inventory and write it to ``config/hosts`` -.. code:: nix - - cp inventory/yaook-k8s/hosts config/hosts - chmod u+w config/hosts - -... and set ``infra.hosts_file = ./hosts;`` in the config. +.. code:: bash -Also remove the ``[gateways]`` section from ``config/hosts`` -and replace ``gateways`` with ``masters`` in the ``[frontend:children]`` section. + yq -y 'del(.gateways) | .frontend.children = {masters: {}}' inventory/yaook-k8s/hosts > config/hosts -We can now disable Terraform: +Now disable OpenStack and import our hosts file into the config: .. code:: nix - terraform.enable = false; + infra.ansible_hosts = yk8s-lib.importYAML ./hosts; openstack.enable = false; Create a jump host diff --git a/docs/user/reference/cluster-configuration.rst b/docs/user/reference/cluster-configuration.rst index 540557125817118d757636c0bbc8096771a8c692..3adf8d639ff4cbe09f421df67af5d52e6903a118 100644 --- a/docs/user/reference/cluster-configuration.rst +++ b/docs/user/reference/cluster-configuration.rst @@ -16,8 +16,7 @@ The cluster repository layout your_cluster_repo ├── config/ # All user configuration now resides in this directory - │ ├── default.nix # Nix-based cluster configuration - │ └── hosts # Manual Ansible hosts file for bare-metal, referenced in default.nix + │ └── default.nix # Nix-based cluster configuration ├── inventory/yaook-k8s/ # Ansible inventory is now completely generated and MAY be excluded from version control │ ├── group-vars/ # Variables passed to Ansible │ └── hosts # Ansible hosts file, generated from config even for bare-metal diff --git a/docs/user/reference/cluster-repository.rst b/docs/user/reference/cluster-repository.rst index 0712e12c9e9f1b298972c62b1af845ddbaab8796..20ed265b99e03f883f8ced3b044eaae33cc4e63f 100644 --- a/docs/user/reference/cluster-repository.rst +++ b/docs/user/reference/cluster-repository.rst @@ -19,7 +19,7 @@ the Tarook cluster. Cluster Repository Structure ---------------------------- -The following schema shows all non-generated files. A local checkout +The following schema shows an example set of files. A local checkout will most certainly have more files than these. :: diff --git a/docs/user/reference/options/yk8s.infra.rst b/docs/user/reference/options/yk8s.infra.rst index 5539a6f2ccbe5ba25cb0d22d79306bee87bf6562..250240b60c798f54aac59df9fb73f6d91262a61a 100644 --- a/docs/user/reference/options/yk8s.infra.rst +++ b/docs/user/reference/options/yk8s.infra.rst @@ -7,6 +7,316 @@ yk8s.infra This section contains various configuration options necessary for all cluster types, Terraform and bare-metal based. +.. _configuration-options.yk8s.infra.ansible_hosts: + +``yk8s.infra.ansible_hosts`` +############################ + +Entries to the Ansible hosts file. Will be rendered to a YAML-based file into the inventory. +This option is mandatory for bare-metal clusters and is automatically managed if Terraform is used. + +Check the parts regarding YAML in the Ansible documentation: https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html + + +**Type:**:: + + null or (attribute set of (submodule)) + + +**Default:**:: + + null + + +**Declared by** +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/infra.nix + + +.. _configuration-options.yk8s.infra.ansible_hosts..children: + +``yk8s.infra.ansible_hosts..children`` +############################################ + + + +**Type:**:: + + attribute set of (submodule) + + +**Default:**:: + + { } + + +**Declared by** + + +.. _configuration-options.yk8s.infra.ansible_hosts..hosts: + +``yk8s.infra.ansible_hosts..hosts`` +######################################### + + + +**Type:**:: + + attribute set of (JSON value) + + +**Default:**:: + + { } + + +**Declared by** + + +.. _configuration-options.yk8s.infra.ansible_hosts..hosts..ansible_host: + +``yk8s.infra.ansible_hosts..hosts..ansible_host`` +############################################################# + + + +**Type:**:: + + null or IPv4 address in four-octets decimal notation or IPv6 address in colon-hexadecimal notation or RFC1123 subdomain name + + +**Default:**:: + + null + + +**Declared by** + + +.. _configuration-options.yk8s.infra.ansible_hosts..hosts..local_ipv4_address: + +``yk8s.infra.ansible_hosts..hosts..local_ipv4_address`` +################################################################### + + + +**Type:**:: + + null or IPv4 address in four-octets decimal notation + + +**Default:**:: + + null + + +**Declared by** + + +.. _configuration-options.yk8s.infra.ansible_hosts..hosts..local_ipv6_address: + +``yk8s.infra.ansible_hosts..hosts..local_ipv6_address`` +################################################################### + + + +**Type:**:: + + null or IPv6 address in colon-hexadecimal notation + + +**Default:**:: + + null + + +**Declared by** + + +.. _configuration-options.yk8s.infra.ansible_hosts..vars: + +``yk8s.infra.ansible_hosts..vars`` +######################################## + + + +**Type:**:: + + attribute set of (JSON value) + + +**Default:**:: + + { } + + +**Declared by** + + +.. _configuration-options.yk8s.infra.ansible_hosts.all.vars.ansible_python_interpreter: + +``yk8s.infra.ansible_hosts.all.vars.ansible_python_interpreter`` +################################################################ + + + +**Type:**:: + + Absolute POSIX path (without special '.' and '..') + + +**Default:**:: + + "/usr/bin/python3" + + +**Declared by** +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/infra.nix + + +.. _configuration-options.yk8s.infra.ansible_hosts.frontend: + +``yk8s.infra.ansible_hosts.frontend`` +##################################### + + + +**Type:**:: + + submodule + + +**Default:**:: + + { + children = { + gateways = { }; + }; + } + + +**Example:**:: + + { + children = { + masters = { }; + }; + } + + +**Declared by** +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/infra.nix + + +.. _configuration-options.yk8s.infra.ansible_hosts.gateways: + +``yk8s.infra.ansible_hosts.gateways`` +##################################### + + + +**Type:**:: + + submodule + + +**Default:**:: + + { } + + +**Declared by** +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/infra.nix + + +.. _configuration-options.yk8s.infra.ansible_hosts.masters: + +``yk8s.infra.ansible_hosts.masters`` +#################################### + + + +**Type:**:: + + submodule + + +**Example:**:: + + { + hosts = { + devcluster-master-1 = { + ansible_host = "172.30.154.66"; + local_ipv4_address = "172.30.154.66"; + }; + }; + } + + +**Declared by** +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/infra.nix + + +.. _configuration-options.yk8s.infra.ansible_hosts.orchestrator: + +``yk8s.infra.ansible_hosts.orchestrator`` +######################################### + + + +**Type:**:: + + submodule + + +**Default:**:: + + { + hosts = { + localhost = { + ansible_connection = "local"; + ansible_python_interpreter = "{{ ansible_playbook_python }}"; + }; + }; + } + + +**Declared by** +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/infra.nix + + +.. _configuration-options.yk8s.infra.ansible_hosts.workers: + +``yk8s.infra.ansible_hosts.workers`` +#################################### + + + +**Type:**:: + + submodule + + +**Default:**:: + + { } + + +**Example:**:: + + { + hosts = { + devcluster-worker-1 = { + ansible_host = "172.30.154.99"; + local_ipv4_address = "172.30.154.99"; + }; + }; + } + + +**Declared by** +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/infra.nix + + .. _configuration-options.yk8s.infra.cluster_name: ``yk8s.infra.cluster_name`` @@ -32,7 +342,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/infra.nix ``yk8s.infra.hosts_file`` ######################### -A custom hosts file in case :ref:`configuration-options.yk8s.openstack.enabled` is set to ``false`` +A custom hosts file. This option is deprecated. Use :ref:`configuration-options.yk8s.infra.ansible_hosts` instead. **Type:**:: diff --git a/nix/yk8s/infra.nix b/nix/yk8s/infra.nix index bc3d81f2f6ab6484d3037caa589b3b70e34eef9c..eb60117bbe91eaea2c860278dc4255915201e65c 100644 --- a/nix/yk8s/infra.nix +++ b/nix/yk8s/infra.nix @@ -10,7 +10,8 @@ inherit (modules-lib) mkRemovedOptionModule; inherit (pkgs.stdenv) mkDerivation; inherit (lib) mkEnableOption mkOption types; - inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkDisableOption linkToPath; + inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkDisableOption linkToPath mkYamlAtPath mkInternalOption; + inherit (yk8s-lib.transform) filterNull; inherit (yk8s-lib.types) ipv4Addr @@ -18,6 +19,9 @@ ipv4Cidr ipv6Cidr k8sClusterName + absolutePosixPath + jsonValue + subdomainName ; in { options.yk8s.infra = mkTopSection { @@ -71,10 +75,8 @@ in { type = types.nullOr ipv4Addr; default = null; apply = v: - if v == null && cfg.ipv4_enabled && config.yk8s.openstack.enabled == false - then throw "config.yk8s.infra.networking_fixed_ip must be set if config.yk8s.infra.ipv4_enabled=true and config.yk8s.openstack.enabled=false" - else if v != null && config.yk8s.openstack.enabled == true - then throw "config.yk8s.infra.networking_fixed_ip must not be set if config.yk8s.openstack.enabled=true" + if cfg.ipv4_enabled && v == null && config.yk8s.terraform.enabled + then builtins.trace "INFO: config.yk8s.infra.networking_fixed_ip is not yet set. Terraform stage needs to be run first." v else v; }; @@ -82,38 +84,259 @@ in { type = with types; nullOr ipv6Addr; default = null; apply = v: - if v == null && cfg.ipv6_enabled && config.yk8s.openstack.enabled == false - then throw "config.yk8s.infra.networking_fixed_ip_v6 must be set if config.yk8s.infra.ipv6_enabled=true and config.yk8s.openstack.enabled=false" - else if v != null && config.yk8s.openstack.enabled == true - then throw "config.yk8s.infra.networking_fixed_ip_v6 must not be set if config.yk8s.openstack.enabled=true" + if cfg.ipv6_enabled && v == null && config.yk8s.terraform.enabled + then builtins.trace "INFO: config.yk8s.infra.networking_fixed_ip_v6 is not yet set. Terraform stage needs to be run first." v + else v; + }; + + networking_floating_ip = mkInternalOption { + # TODO: move to yk8s.wireguard when ipsec gets removed + description = '' + Address that is used by Wireguard and IPsec to connect to the active gateway node. + ''; + type = types.nullOr ipv4Addr; + default = null; + apply = v: + if v == null && config.yk8s.terraform.enabled + then builtins.trace "INFO: config.yk8s.infra.networking_floating_ip is not yet set. Terraform stage needs to be run first." v else v; }; hosts_file = mkOption { description = '' - A custom hosts file in case :ref:`configuration-options.yk8s.openstack.enabled` is set to ``false`` + A custom hosts file. This option is deprecated. Use :ref:`configuration-options.yk8s.infra.ansible_hosts` instead. ''; type = with types; nullOr pathInStore; default = null; example = lib.options.literalExpression "./hosts"; - apply = v: - if v == null && config.yk8s.openstack.enabled == false - then throw "infra.hosts_file must be set if openstack is disabled" - else if v != null && config.yk8s.openstack.enabled == true - then throw "infra.hosts_file must not be set if openstack is enabled" - else v; + }; + + ansible_hosts = let + applyGroupSubmoduleAttrs = lib.mapAttrs (_: lib.filterAttrs (_: a: a != {})); + hostsSubmodule = types.submodule { + freeformType = jsonValue; + options = { + ansible_host = mkOption { + type = with types; nullOr (oneOf [ipv4Addr ipv6Addr subdomainName]); + default = null; + }; + local_ipv4_address = mkOption { + type = types.nullOr ipv4Addr; + default = null; + }; + local_ipv6_address = mkOption { + type = types.nullOr ipv6Addr; + default = null; + }; + }; + }; + groupSubmodule = types.submodule { + options = { + children = mkOption { + visible = "shallow"; # Otherwise renderDocs chokes on the recursive submodule + type = types.attrsOf groupSubmodule; + default = {}; + apply = applyGroupSubmoduleAttrs; + }; + hosts = mkOption { + type = types.attrsOf hostsSubmodule; + default = {}; + }; + vars = mkOption { + type = types.attrsOf jsonValue; + default = {}; + }; + }; + }; + in + mkOption { + description = '' + Entries to the Ansible hosts file. Will be rendered to a YAML-based file into the inventory. + This option is mandatory for bare-metal clusters and is automatically managed if Terraform is used. + + Check the parts regarding YAML in the Ansible documentation: https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html + ''; + default = null; + apply = v: + if v == null && config.yk8s.terraform.enabled + then builtins.trace "INFO: infra.ansible_hosts is not yet set. Terraform stage needs to be run first." v + else applyGroupSubmoduleAttrs v; + type = types.nullOr (types.submodule { + freeformType = types.attrsOf groupSubmodule; + options = { + all.vars.ansible_python_interpreter = mkOption { + type = absolutePosixPath; + default = "/usr/bin/python3"; + }; + frontend = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + default = {children.gateways = {};}; + example = {children.masters = {};}; + }; + gateways = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + default = {}; + }; + k8s_nodes = mkInternalOption { + readOnly = true; + type = groupSubmodule; + default = { + children = { + masters = {}; + workers = {}; + }; + }; + }; + masters = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + example = { + hosts = { + devcluster-master-1 = { + ansible_host = "172.30.154.66"; + local_ipv4_address = "172.30.154.66"; + }; + }; + }; + }; + workers = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + default = {}; + example = { + hosts = { + devcluster-worker-1 = { + ansible_host = "172.30.154.99"; + local_ipv4_address = "172.30.154.99"; + }; + }; + }; + }; + orchestrator = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + default = { + hosts.localhost = { + ansible_connection = "local"; + ansible_python_interpreter = "{{ ansible_playbook_python }}"; + }; + }; + }; + }; + }); + }; + + final_hosts = mkInternalOption { + description = '' + Internal read-only option to access all hosts and their effective attributes available to Ansible. + Each host gets the following additional attributes + * ``group_names``: A list of all Ansible groups to which the host belongs + * ``role``: One of ``master``, ``worker``, ``gateway`` or ``null`` + ''; + readOnly = true; + type = with types; nullOr attrs; + default = + if cfg.ansible_hosts == null + then null + else let + getHostsFromGroupAttrs = lib.foldlAttrs ( + acc: groupName: groupValues: let + hosts = lib.recursiveUpdate (getHostsFromGroupAttrs ( + lib.filterAttrs (n: _: builtins.elem n (builtins.attrNames (groupValues.children or {}))) cfg.ansible_hosts + )) (groupValues.hosts or {}); + in + lib.recursiveUpdate acc (lib.mapAttrs ( + hostName: hostValues: + hostValues + // rec { + group_names = lib.unique ((lib.attrByPath [hostName "group_names"] [] acc) ++ [groupName]); + role = let + relevantGroups = lib.intersectLists group_names ["masters" "workers" "gateways"]; + in + assert lib.assertMsg ((builtins.length relevantGroups) <= 1) "${hostName} has more than one role assigned. Nodes can only be one of master, worker or gateway"; + if relevantGroups == [] + then null + else lib.strings.removeSuffix "s" (builtins.head relevantGroups); + } + ) + hosts) + ) {}; + allHosts = getHostsFromGroupAttrs cfg.ansible_hosts; + groupNames = lib.pipe allHosts [builtins.attrValues (map (v: v.group_names)) lib.flatten lib.unique]; + allGroups = builtins.foldl' lib.recursiveUpdate cfg.ansible_hosts ([{all.hosts = allHosts;}] + ++ (map (group: { + ${group} = { + children = cfg.ansible_hosts.${group}.children or {}; + vars = cfg.ansible_hosts.${group}.vars or {}; + hosts = lib.filterAttrs (_: v: builtins.elem group v.group_names) allHosts; + }; + }) + groupNames)); + populateChildren = groupName: groupValues: + groupValues + // {children = lib.mapAttrs (childName: _: allGroups.${childName}) (groupValues.children or {});}; + in + lib.mapAttrs populateChildren allGroups; }; }; + + config.yk8s.assertions = [ + { + assertion = + (cfg.ansible_hosts != null) + -> (cfg.ansible_hosts.orchestrator.children or {}) == {} && (builtins.length (builtins.attrNames cfg.ansible_hosts.orchestrator.hosts)) == 1; + message = "config.yk8s.infra.ansible_hosts.orchestrator must contain exactly one host and no children"; + } + { + assertion = cfg.ipv4_enabled -> config.yk8s.terraform.enabled || cfg.networking_fixed_ip != null; + message = "config.yk8s.infra.networking_fixed_ip must be set if Terraform is not used"; + } + { + assertion = cfg.ipv6_enabled -> config.yk8s.terraform.enabled || cfg.networking_fixed_ip_v6 != null; + message = "config.yk8s.infra.networking_fixed_ip_v6 must be set if Terraform is not used"; + } + { + assertion = (config.yk8s.wireguard.enabled || config.yk8s.ipsec.enabled) -> config.yk8s.terraform.enabled || cfg.networking_floating_ip != null; + message = "config.yk8s.infra.networking_floating_ip must be set if Wireguard or IPsec is used."; + } + { + assertion = cfg.ansible_hosts != null -> cfg.hosts_file == null; + message = "config.yk8s.infra.hosts_file must not be set if config.yk8s.infra.ansible_hosts is used (which implicitly happens through Terraform)."; + } + { + assertion = ! config.yk8s.terraform.enabled -> (cfg.ansible_hosts == null && cfg.hosts_file == null); + message = "One of config.yk8s.infra.hosts_file and config.yk8s.infra.ansible_hosts must be set"; + } + { + assertion = + (cfg.ansible_hosts != null) + -> builtins.all (host: (host.ansible_connection or "") != "local" -> host.ansible_host != null) (builtins.attrValues cfg.final_hosts.all.hosts); + message = "ansible_host must be set for all hosts in config.yk8s.infra.ansible_hosts if ansible_connection!=local"; + } + { + assertion = + (cfg.ansible_hosts != null && cfg.ipv4_enabled) + -> builtins.all (host: host.local_ipv4_address != null) (builtins.attrValues cfg.final_hosts.k8s_nodes.hosts); + message = "local_ipv4_address must be set for all hosts in config.yk8s.infra.ansible_hosts.k8s_nodes"; + } + { + assertion = + (cfg.ansible_hosts != null && cfg.ipv6_enabled) + -> builtins.all (host: host.local_ipv6_address != null) (builtins.attrValues cfg.final_hosts.k8s_nodes.hosts); + message = "local_ipv6_address must be set for all hosts in config.yk8s.infra.ansible_hosts.k8s_nodes"; + } + ]; + config.yk8s.warnings = lib.optional (cfg.hosts_file != null) "config.yk8s.infra.hosts_file is deprecated. Use config.yk8s.infra.ansible_hosts instead."; config.yk8s._inventory_packages = - [ + (lib.optional (cfg.ansible_hosts != null) (mkYamlAtPath "hosts" (filterNull cfg.ansible_hosts))) + ++ (lib.optional (cfg.hosts_file != null) (linkToPath cfg.hosts_file "hosts")) + ++ [ (mkGroupVarsFile { inherit cfg; inventory_path = "all/infra.yaml"; - transformations = - [(lib.attrsets.filterAttrs (n: _: n != "hosts_file"))] - ++ (lib.optional config.yk8s.openstack.enabled (lib.attrsets.filterAttrs (n: _: ! (builtins.elem n ["networking_fixed_ip" "networking_fixed_ip_v6"])))); + transformations = [(c: removeAttrs c ["hosts_file" "ansible_hosts" "final_hosts"])]; }) - ] - ++ lib.optional (cfg.hosts_file != null) - (linkToPath cfg.hosts_file "hosts"); + ]; } diff --git a/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix b/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix index 5b28793e7eeb7a802c50cbe0930548885e3d2abf..28e9dce9e0d641d0abde679d1e41d06895dc9c62 100644 --- a/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix +++ b/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix @@ -8,7 +8,7 @@ modules-lib = import ../lib/modules.nix {inherit lib;}; inherit (modules-lib) mkRenamedResourceOptionModule mkResourceOptionModule; inherit (lib) mkOption mkEnableOption types; - inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkResourceOption mkDisableOption; + inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkInternalOption mkDisableOption; inherit (yk8s-lib.types) base64Str @@ -158,6 +158,22 @@ in { Be aware, that the frontend nodes must be potent enough to handle the increased amount of traffic if source-nat'ing is disabled, as they could become the bottleneck otherwise''; + subnet_id = mkInternalOption { + type = with types; nullOr nonEmptyStr; + default = null; + apply = v: + if v == null && config.yk8s.terraform.enabled + then builtins.trace "INFO: ch-k8s-lbaas.subnet_id is not yet set. Terraform stage needs to be run first." v + else v; + }; + floating_ip_network_id = mkInternalOption { + type = with types; nullOr nonEmptyStr; + default = null; + apply = v: + if v == null && config.yk8s.terraform.enabled + then builtins.trace "INFO: ch-k8s-lbaas.floating_ip_network_id is not yet set. Terraform stage needs to be run first." v + else v; + }; }; config.yk8s.assertions = [ # Due to OVN support, require version >= 0.8.0 (warn only if not in semver2 format) @@ -203,6 +219,10 @@ in { message = "config.yk8s.ch-k8s-lbaas.version: must be at least 0.8.0"; } ) + { + assertion = (!config.yk8s.openstack.enabled) -> (cfg.subnet_id == null && cfg.floating_ip_network_id == null); + message = "config.yk8s.ch-k8s-lbaas.subnet_id and config.yk8s.ch-k8s-lbaas.floating_ip_network_id must be null if config.yk8s.openstack.enabled==false"; + } ]; config.yk8s._inventory_packages = [ (mkGroupVarsFile { diff --git a/nix/yk8s/node-scheduling.nix b/nix/yk8s/node-scheduling.nix index 3a44f767ed06652ffc4b8b2e6c14360f4e453dce..6037ea55b8f69c641b9119ed19d05062310a587f 100644 --- a/nix/yk8s/node-scheduling.nix +++ b/nix/yk8s/node-scheduling.nix @@ -8,7 +8,6 @@ cfg = config.yk8s.node-scheduling; inherit (lib) mkOption types; inherit (yk8s-lib) mkTopSection mkGroupVarsFile; - nodeNames = map (n: "${config.yk8s.infra.cluster_name}-${n}") (builtins.attrNames config.yk8s.openstack.nodes); inherit (yk8s-lib.types) k8sLabelStr @@ -71,12 +70,6 @@ in { "''${scheduling_key_prefix}/monitoring=true" ]; }''; - apply = v: - builtins.seq (builtins.all (e: - if config.yk8s.terraform.enabled -> builtins.elem e nodeNames - then true - else throw "config.yk8s.node-scheduling.labels: label defined for ${e}, but node not found in config.yk8s.openstack.nodes") (builtins.attrNames v)) - v; }; taints = mkOption { description = '' @@ -96,14 +89,17 @@ in { "''${scheduling_key_prefix}/storage=true:NoSchedule" ]; }''; - apply = v: - builtins.seq (builtins.all (e: - if config.yk8s.terraform.enabled -> builtins.elem e nodeNames - then true - else throw "config.yk8s.node-scheduling.taints: taint defined for ${e}, but node not found in config.yk8s.openstack.nodes") (builtins.attrNames v)) - v; }; }; + config.yk8s.warnings = + (builtins.foldl' (acc: e: + acc + ++ lib.optional (config.yk8s.infra.final_hosts != null && ! builtins.hasAttr e (config.yk8s.infra.final_hosts.all.hosts or {})) + "config.yk8s.node-scheduling.labels: label defined for ${e}, but node not found in config.yk8s.infra.ansible_hosts") [] (builtins.attrNames cfg.labels)) + ++ (builtins.foldl' (acc: e: + acc + ++ lib.optional (config.yk8s.infra.final_hosts != null && ! builtins.hasAttr e (config.yk8s.infra.final_hosts.all.hosts or {})) + "config.yk8s.node-scheduling.taints: taint defined for ${e}, but node not found in config.yk8s.infra.ansible_hosts") [] (builtins.attrNames cfg.taints)); config.yk8s._inventory_packages = [ (mkGroupVarsFile { inherit cfg; diff --git a/nix/yk8s/openstack.nix b/nix/yk8s/openstack.nix index b9a82666cecb5d536c621c6b2eec888593ca9504..a480c4c887b00403ec745349129160ab8a5e99a7 100644 --- a/nix/yk8s/openstack.nix +++ b/nix/yk8s/openstack.nix @@ -350,6 +350,17 @@ in { ''; }; config.yk8s = lib.mkMerge [ + { + _inventory_packages = [ + (mkGroupVarsFile { + inherit cfg; + inventory_path = "all/openstack.yaml"; + ansible_prefix = "openstack_"; + only_if_enabled = true; + transformations = [(c: builtins.removeAttrs c nonAnsibleOptions)]; + }) + ]; + } (lib.mkIf cfg.enabled { terraform.enabled = true; @@ -408,17 +419,67 @@ in { message = "config.yk8s.openstack.network_mtu: must be at least 1280 Bytes to support IPv6."; } ]; + + infra = { + networking_floating_ip = config.yk8s.terraform.outputs.networking_floating_ip.value or null; + networking_fixed_ip = config.yk8s.terraform.outputs.networking_fixed_ip.value or null; + networking_fixed_ip_v6 = config.yk8s.terraform.outputs.networking_fixed_ip_v6.value or null; + ansible_hosts = + if config.yk8s.terraform.outputs == null + then null + else { + all.vars = {}; + + frontend.children = { + gateways = {}; + }; + + gateways.hosts = + lib.mapAttrs ( + name: _: + { + ansible_host = config.yk8s.terraform.outputs.gateway_fips.value.${name}.address; + port_id = config.yk8s.terraform.outputs.gateway_ports.value.${name}.id; + local_ipv4_address = builtins.head config.yk8s.terraform.outputs.gateway_ports.value.${name}.all_fixed_ips; + } + // lib.optionalAttrs config.yk8s.infra.ipv6_enabled { + local_ipv6_address = builtins.elemAt config.yk8s.terraform.outputs.gateway_ports.value.${name}.all_fixed_ips 1; + } + ) + config.yk8s.terraform.outputs.gateways.value; + + masters.hosts = + lib.mapAttrs ( + name: _: + { + ansible_host = builtins.head config.yk8s.terraform.outputs.master_ports.value.${name}.all_fixed_ips; + port_id = config.yk8s.terraform.outputs.master_ports.value.${name}.id; + local_ipv4_address = builtins.head config.yk8s.terraform.outputs.master_ports.value.${name}.all_fixed_ips; + } + // lib.optionalAttrs config.yk8s.infra.ipv6_enabled { + local_ipv6_address = builtins.elemAt config.yk8s.terraform.outputs.master_ports.value.${name}.all_fixed_ips 1; + } + ) + config.yk8s.terraform.outputs.masters.value; + workers.hosts = + lib.mapAttrs ( + name: _: + { + ansible_host = builtins.head config.yk8s.terraform.outputs.worker_ports.value.${name}.all_fixed_ips; + port_id = config.yk8s.terraform.outputs.worker_ports.value.${name}.id; + local_ipv4_address = builtins.head config.yk8s.terraform.outputs.worker_ports.value.${name}.all_fixed_ips; + } + // lib.optionalAttrs config.yk8s.infra.ipv6_enabled { + local_ipv6_address = builtins.elemAt config.yk8s.terraform.outputs.worker_ports.value.${name}.all_fixed_ips 1; + } + ) + config.yk8s.terraform.outputs.workers.value; + }; + }; + ch-k8s-lbaas = { + subnet_id = config.yk8s.terraform.outputs.subnet_id.value or null; + floating_ip_network_id = config.yk8s.terraform.outputs.floating_ip_network_id.value or null; + }; }) - { - _inventory_packages = [ - (mkGroupVarsFile { - inherit cfg; - inventory_path = "all/openstack.yaml"; - ansible_prefix = "openstack_"; - only_if_enabled = true; - transformations = [(c: builtins.removeAttrs c nonAnsibleOptions)]; - }) - ]; - } ]; } diff --git a/nix/yk8s/terraform.nix b/nix/yk8s/terraform.nix index 3bdb1c8b7e74e8d12363f02419d1284f03d4ef9a..924e05dea4f55d3b97cf2af6958d336092acf52e 100644 --- a/nix/yk8s/terraform.nix +++ b/nix/yk8s/terraform.nix @@ -175,33 +175,31 @@ in { default = null; example = "tf-state"; }; + + outputs = mkInternalOption { + readOnly = true; + type = with types; nullOr attrs; + default = let + tfOutputsPath = "terraform/outputs.json"; + tfOutputsFullPath = "${config.yk8s.state_directory}/${tfOutputsPath}"; + in + if config.yk8s.state_directory != null && builtins.pathExists tfOutputsFullPath + then builtins.fromJSON (builtins.readFile tfOutputsFullPath) + else null; + }; }; config.yk8s = { - _inventory_packages = - [ - (mkGroupVarsFile { - cfg = lib.attrsets.getAttrs ["enabled"] cfg; - inventory_path = "all/terraform.yaml"; - }) - ] - ++ lib.optionals cfg.enabled ( - let - linkTfstateIfExists = source: target: - if config.yk8s.state_directory != null && builtins.pathExists "${config.yk8s.state_directory}/${source}" - then [(linkToPath "${config.yk8s.state_directory}/${source}" target)] - else - builtins.trace "INFO: ${config.yk8s._state_base_path}/${source} does not yet exist. Terraform stage needs to be run first." - []; - in - (linkTfstateIfExists "terraform/rendered/hosts" "hosts") - ++ (linkTfstateIfExists "terraform/rendered/terraform_networking-trampoline.yaml" "group_vars/all/terraform_networking-trampoline.yaml") - ++ (linkTfstateIfExists "terraform/rendered/terraform_networking.yaml" "group_vars/all/terraform_networking.yaml") - ); + _inventory_packages = [ + (mkGroupVarsFile { + cfg = lib.attrsets.getAttrs ["enabled"] cfg; + inventory_path = "all/terraform.yaml"; + }) + ]; _state_packages = lib.optional cfg.enabled ( let - filteredTerraformCfg = yk8s-lib.removeAttrsByPath config.yk8s.terraform [["enabled"]]; + filteredTerraformCfg = yk8s-lib.removeAttrsByPath config.yk8s.terraform [["enabled"] ["outputs"]]; filteredInfraCfg = lib.attrsets.getAttrs infraTerraformOptions config.yk8s.infra; filteredOpenstackCfg = lib.attrsets.getAttrs openstackTerraformOptions config.yk8s.openstack; mergedCfg = diff --git a/terraform/01-discovery.tf b/terraform/01-discovery.tf index 29a28bb92ec25f2f1744687fda8ce6716eec4bbb..0187b987e533f618fb645a3199349dd5a8506293 100644 --- a/terraform/01-discovery.tf +++ b/terraform/01-discovery.tf @@ -10,3 +10,7 @@ data "openstack_images_image_v2" "gateway" { name = var.gateway_defaults.image most_recent = true } + +output floating_ip_network_id { + value = data.openstack_networking_network_v2.public_network.id +} diff --git a/terraform/10-networking.tf b/terraform/10-networking.tf index 990631952f05341d4f1c743c4df97d6c932831ab..9c4c490b0bace766a1f41ac318e46079e52205a6 100644 --- a/terraform/10-networking.tf +++ b/terraform/10-networking.tf @@ -270,3 +270,11 @@ resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-vrrp-egress" { protocol = "vrrp" security_group_id = openstack_networking_secgroup_v2.barndoor.id } + +output subnet_id { + value = try(openstack_networking_subnet_v2.cluster_subnet[0].id, null) +} + +output subnet_v6_id { + value = try(openstack_networking_subnet_v2.cluster_v6_subnet[0].id, null) +} diff --git a/terraform/20-gateway.tf b/terraform/20-gateway.tf index c010c1007d9046a064d870ae5fee469c1d92c0cd..de8a26ae1b0ce8c7f0c140ada8e6e542deaacd97 100644 --- a/terraform/20-gateway.tf +++ b/terraform/20-gateway.tf @@ -190,11 +190,26 @@ resource "openstack_networking_floatingip_associate_v2" "gateway" { ] } -data "template_file" "trampoline_gateways" { - template = file("${path.module}/templates/trampoline_gateways.tpl") - vars = { - networking_fixed_ip = try(jsonencode(openstack_networking_port_v2.gw_vip_port.all_fixed_ips[0]), "null"), - networking_fixed_ip_v6 = try(jsonencode(openstack_networking_port_v2.gw_vip_port.all_fixed_ips[1]), "null"), - networking_floating_ip = openstack_networking_floatingip_v2.gw_vip_fip.address, - } +output gateways { + value = openstack_compute_instance_v2.gateway + sensitive = true +} +output gateway_ports { + value = openstack_networking_port_v2.gateway +} +output gateway_fips { + value = openstack_networking_floatingip_v2.gateway +} + +output networking_fixed_ip { + value = try(openstack_networking_port_v2.gw_vip_port.all_fixed_ips[0], null) +} + +output networking_fixed_ip_v6 { + value = try(openstack_networking_port_v2.gw_vip_port.all_fixed_ips[1], null) + +} + +output networking_floating_ip { + value = openstack_networking_floatingip_v2.gw_vip_fip.address } diff --git a/terraform/30-master-nodes.tf b/terraform/30-master-nodes.tf index 821a05776339bd9580933da694f9fd0d208f3b3b..307b5cdcc1ab051376eb69f2ff0785aac86d8c91 100644 --- a/terraform/30-master-nodes.tf +++ b/terraform/30-master-nodes.tf @@ -97,3 +97,11 @@ resource "openstack_compute_instance_v2" "master" { ignore_changes = [key_pair, image_id, config_drive] } } + +output masters { + value = openstack_compute_instance_v2.master + sensitive = true +} +output master_ports { + value = openstack_networking_port_v2.master +} diff --git a/terraform/40-worker-nodes.tf b/terraform/40-worker-nodes.tf index 2c96cc6097571c98e0cdc8ad059aa31814bf9925..ad96bd0c7bd716303b2ee9685492681eaaeec9e9 100644 --- a/terraform/40-worker-nodes.tf +++ b/terraform/40-worker-nodes.tf @@ -117,3 +117,11 @@ resource "openstack_compute_instance_v2" "worker" { ignore_changes = [key_pair, image_id, config_drive, scheduler_hints] } } + +output workers { + value = openstack_compute_instance_v2.worker + sensitive = true +} +output worker_ports { + value = openstack_networking_port_v2.worker +} diff --git a/terraform/99-backend.tf b/terraform/99-backend.tf new file mode 100644 index 0000000000000000000000000000000000000000..4182b864765b6e348c02bdb0d79852133c66c13e --- /dev/null +++ b/terraform/99-backend.tf @@ -0,0 +1,7 @@ +# Please note that if gitlab_backend is set to true in the config +# it will override this local backend configuration +terraform { + backend "local" { + path = "../../state/terraform/terraform.tfstate" + } +} diff --git a/terraform/99-outputs.tf b/terraform/99-outputs.tf deleted file mode 100644 index abc32b768c9e9588586dda5cac5227a0a41c0f70..0000000000000000000000000000000000000000 --- a/terraform/99-outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -resource "local_file" "inventory_yaook-k8s" { - content = templatefile("${path.module}/templates/inventory.tpl", { - masters = openstack_compute_instance_v2.master, - master_ports = openstack_networking_port_v2.master, - gateways = openstack_compute_instance_v2.gateway, - gateway_ports = openstack_networking_port_v2.gateway, - gateway_fips = openstack_networking_floatingip_v2.gateway, - workers = openstack_compute_instance_v2.worker, - worker_ports = openstack_networking_port_v2.worker, - ipv6_enabled = var.ipv6_enabled, - ipv4_enabled = var.ipv4_enabled, - }) - filename = "../../state/terraform/rendered/hosts" - file_permission = 0640 -} - -resource "local_file" "trampoline_gateways" { - content = data.template_file.trampoline_gateways.rendered - filename = "../../state/terraform/rendered/terraform_networking-trampoline.yaml" - file_permission = 0640 -} - -resource "local_file" "final_networking" { - content = templatefile("${path.module}/templates/final_networking.tpl", { - subnet_id = try(openstack_networking_subnet_v2.cluster_subnet[0].id, null), - subnet_v6_id = try(openstack_networking_subnet_v2.cluster_v6_subnet[0].id, null), - floating_ip_network_id = data.openstack_networking_network_v2.public_network.id, - }) - filename = "../../state/terraform/rendered/terraform_networking.yaml" - file_permission = 0640 -} - -# Please note that if gitlab_backend is set to true in the config -# it will override this local backend configuration -terraform { - backend "local" { - path = "../../state/terraform/terraform.tfstate" - } -} diff --git a/terraform/templates/final_networking.tpl b/terraform/templates/final_networking.tpl deleted file mode 100644 index 35f0be5555016542ec28e865a6d5148a1f1b07d8..0000000000000000000000000000000000000000 --- a/terraform/templates/final_networking.tpl +++ /dev/null @@ -1,4 +0,0 @@ -openstack_lbaas_subnet_id: ${jsonencode(subnet_id)} -openstack_lbaas_floating_ip_network_id: ${jsonencode(floating_ip_network_id)} -ch_k8s_lbaas_subnet_id: ${jsonencode(subnet_id)} -ch_k8s_lbaas_floating_ip_network_id: ${jsonencode(floating_ip_network_id)} diff --git a/terraform/templates/inventory.tpl b/terraform/templates/inventory.tpl deleted file mode 100644 index 46c1418e045c2844d7c9af313799bd693a10a382..0000000000000000000000000000000000000000 --- a/terraform/templates/inventory.tpl +++ /dev/null @@ -1,27 +0,0 @@ -[all:vars] -ansible_python_interpreter=/usr/bin/python3 - -[orchestrator] -localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}" - -[frontend:children] -gateways - -[k8s_nodes:children] -masters -workers - -[gateways] -%{ for index, instance in gateways ~} -${instance.name} ansible_host=${gateway_fips[index].address} port_id=${gateway_ports[index].id} local_ipv4_address=${gateway_ports[index].all_fixed_ips[0]} %{if ipv6_enabled }${try("local_ipv6_address=${gateway_ports[index].all_fixed_ips[1]}", "")}%{ endif } -%{ endfor } - -[masters] -%{ for index, instance in masters ~} -${instance.name} ansible_host=${master_ports[index].all_fixed_ips[0]} local_ipv4_address=${master_ports[index].all_fixed_ips[0]} %{if ipv6_enabled }${try("local_ipv6_address=${master_ports[index].all_fixed_ips[1]}", "")}%{ endif } -%{ endfor } - -[workers] -%{ for index, instance in workers ~} -${instance.name} ansible_host=${worker_ports[index].all_fixed_ips[0]} local_ipv4_address=${worker_ports[index].all_fixed_ips[0]} %{if ipv6_enabled }${try("local_ipv6_address=${worker_ports[index].all_fixed_ips[1]}", "")}%{ endif } -%{ endfor } diff --git a/terraform/templates/object_storage.tpl b/terraform/templates/object_storage.tpl deleted file mode 100644 index 1369dbf91ee6ded0e12bbc69f27f13eb15be2794..0000000000000000000000000000000000000000 --- a/terraform/templates/object_storage.tpl +++ /dev/null @@ -1 +0,0 @@ -monitoring_thanos_objectstorage_container_name: ${monitoring_thanos_objectstorage_container_name} diff --git a/terraform/templates/trampoline_gateways.tpl b/terraform/templates/trampoline_gateways.tpl deleted file mode 100644 index 9c74d3a92554f54a14945acc3e6e820a8961cfb5..0000000000000000000000000000000000000000 --- a/terraform/templates/trampoline_gateways.tpl +++ /dev/null @@ -1,3 +0,0 @@ -networking_floating_ip : ${networking_floating_ip} -networking_fixed_ip : ${networking_fixed_ip} -networking_fixed_ip_v6: ${networking_fixed_ip_v6}