diff --git a/docs/_releasenotes/1943.chore.1.bird-compose-filter b/docs/_releasenotes/1943.chore.1.bird-compose-filter new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/docs/_releasenotes/1943.chore.2.ch-k8s-lbaas-dependencies b/docs/_releasenotes/1943.chore.2.ch-k8s-lbaas-dependencies new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/docs/_releasenotes/1943.feature.1.no-snat b/docs/_releasenotes/1943.feature.1.no-snat new file mode 100644 index 0000000000000000000000000000000000000000..3fb2269febeb20b95a3fb23e0a8286e2bc0d144d --- /dev/null +++ b/docs/_releasenotes/1943.feature.1.no-snat @@ -0,0 +1,30 @@ +An option to disable SNAT'ing for ``ch-k8s-lbaas`` has been added: +:ref:`configuration-options.yk8s.ch-k8s-lbaas.enable_snat`. + +.. warning:: + + Be aware that disabling SNAT'ing potentially has performance implications. + +.. tabs:: + + .. tab:: Rollout immediately after release upgrade + + When directly coming from a previous release and you want to disable SNAT'ing right away + without having done a full rollout yet, + you have to adjust the gateway nodes first. + In that case, rollout the necessary changes with: + + .. code:: console + + $ ./managed-k8s/actions/apply-prepare-gw.sh + $ ./managed-k8s/actions/apply-k8s-supplements.sh install-ch-k8s-lbaas.yaml + + .. tab:: Rollout later + + If you already did a rollout with the current release, it's sufficient to do: + + .. code:: console + + $ ./managed-k8s/actions/apply-k8s-supplements.sh install-ch-k8s-lbaas.yaml + +. diff --git a/docs/user/reference/options/yk8s.ch-k8s-lbaas.rst b/docs/user/reference/options/yk8s.ch-k8s-lbaas.rst index bf098fce7eb7b890290d4b27ea7c41b012e08206..1e7b03759357c5f998713a6366729228687a114a 100644 --- a/docs/user/reference/options/yk8s.ch-k8s-lbaas.rst +++ b/docs/user/reference/options/yk8s.ch-k8s-lbaas.rst @@ -235,6 +235,64 @@ Thus, this option is deprecated. https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix +.. _configuration-options.yk8s.ch-k8s-lbaas.enable_snat: + +``yk8s.ch-k8s-lbaas.enable_snat`` +################################# + +Whether to enable source-nat'ing by the ch-k8s-lbaas-agents running on the frontend nodes. + +Disabling this has a similar effect as a direct server return. +It allows to see the real source IP of traffic sent to a LoadBalancer-service. + +After reconfiguring this option, execute the following: + +.. code:: + + $ ./managed-k8s/actions/apply-k8s-supplements.sh install-ch-k8s-lbaas.yaml + +to rollout necessary changes. + +Running on OpenStack +"""""""""""""""""""" + +If source-nat'ing is disabled, the frontend nodes will be configured to act as gateway +for the Kubernetes nodes. They will propagate routes via BGP overwriting the default routes of +Kubernetes nodes such that **all** traffic is routed via the VIP by default. + +.. warning:: Implications when running on OpenStack + + Disabling source-nat'ing has some implications: + + 1. If a failover occurs on the frontend nodes, **all** connections are impacted, + not only connections to LoadBalancer-Services. + 2. The source IP of Kubernetes nodes as seen by the outside world changes from + the OpenStack router IP to the Gateway's VIP. + 3. It's not possible to attach floating IPs to Kubernetes nodes anymore due to + routing asymmetry. + +Be aware, that the frontend nodes must be potent enough to handle the increased amount of traffic +if source-nat'ing is disabled, as they could become the bottleneck otherwise. + +**Type:**:: + + boolean + + +**Default:**:: + + true + + +**Example:**:: + + false + + +**Declared by** +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix + + .. _configuration-options.yk8s.ch-k8s-lbaas.enabled: ``yk8s.ch-k8s-lbaas.enabled`` diff --git a/k8s-supplements/ansible/roles/bird/templates/bird-calico-v6.conf.j2 b/k8s-supplements/ansible/roles/bird/templates/bird-calico-v6.conf.j2 index 905be12b661a140f73cc834edeaafee4b4c1b41b..3a179c13658e284999e0b1e494d8b26cbaad5a63 100644 --- a/k8s-supplements/ansible/roles/bird/templates/bird-calico-v6.conf.j2 +++ b/k8s-supplements/ansible/roles/bird/templates/bird-calico-v6.conf.j2 @@ -11,6 +11,19 @@ protocol static k8s_service_net { export none; } +function filter_organization() { + if scope = SCOPE_ORGANIZATION then return true; + return false; +} + +filter k8s_worker_v6 { +{% if wg_enabled %} + if filter_wireguard_v6() then accept; +{% endif %} + if filter_organization() then accept; + reject; +} + {% for hostname in groups['k8s_nodes'] %} {% set peer = hostvars[hostname] %} protocol bgp {{ "%s_v6" | format(hostname | replace('-', '_')) }} { diff --git a/k8s-supplements/ansible/roles/bird/templates/bird-calico.conf.j2 b/k8s-supplements/ansible/roles/bird/templates/bird-calico.conf.j2 index 2ea6aa3ce4b5648d0a8c65b7f7efa3311ccfc560..76083cf89580d23d95360e6d1ce3ef62ea1ef701 100644 --- a/k8s-supplements/ansible/roles/bird/templates/bird-calico.conf.j2 +++ b/k8s-supplements/ansible/roles/bird/templates/bird-calico.conf.j2 @@ -11,6 +11,22 @@ protocol static k8s_service_net { export none; } +function filter_organization() { + if scope = SCOPE_ORGANIZATION then return true; + return false; +} + +filter k8s_worker { +{% if ch_k8s_lbaas_enabled and not ch_k8s_lbaas_enable_snat %} + if filter_default_overwrite() then accept; +{% endif %} +{% if wg_enabled %} + if filter_wireguard() then accept; +{% endif %} + if filter_organization() then accept; + reject; +} + {% for hostname in groups['k8s_nodes'] %} {% set peer = hostvars[hostname] %} protocol bgp {{ hostname | replace('-', '_') }} { diff --git a/k8s-supplements/ansible/roles/bird/templates/bird.conf.j2 b/k8s-supplements/ansible/roles/bird/templates/bird.conf.j2 index 42655d989e087b180e08d12017a64019094ddb4f..1bc571bb36381471c635c1066f8e04231291283a 100644 --- a/k8s-supplements/ansible/roles/bird/templates/bird.conf.j2 +++ b/k8s-supplements/ansible/roles/bird/templates/bird.conf.j2 @@ -7,7 +7,14 @@ protocol kernel { table master; scan time 60; import none; +{% if ch_k8s_lbaas_enabled and not ch_k8s_lbaas_enable_snat %} + export filter { + if net = 0.0.0.0/1 || net = 128.0.0.0/1 then reject; + accept; + }; +{% else %} export all; +{% endif%} } protocol direct { diff --git a/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/meta/main.yaml b/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/meta/main.yaml index c30b24ca2f5b770514b7486126da6d5bbd60e896..2f410663450af98902439f99538902bbd0686fbb 100644 --- a/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/meta/main.yaml +++ b/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/meta/main.yaml @@ -5,5 +5,4 @@ galaxy_info: dependencies: - config/k8s-config - ch-k8s-lbaas-common -- bird ... diff --git a/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/tasks/main.yaml b/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/tasks/main.yaml index 7148418cbce7ed4e9dd190cf2337607abedcfe89..7f49a5e6bf95eb5217ecfaa7727f19799e50a830 100644 --- a/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/tasks/main.yaml +++ b/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/tasks/main.yaml @@ -1,4 +1,14 @@ --- +- name: Import dependency roles needed when running on OpenStack + when: openstack_enabled + block: + - name: Import bird + import_role: + name: bird + - name: Import nftables + import_role: + name: nftables + - name: Deploy LBaaS agent become: true block: @@ -76,6 +86,23 @@ notify: - restart ch-k8s-lbaas-agent + - name: Setup no-SNAT configuration + template: + src: bird.no-snat.conf.j2 + dest: /etc/bird.d/01-no-snat.conf + owner: root + group: bird + mode: "u=rw,g=r,o-rwx" + notify: restart bird + when: openstack_enabled and ipv4_enabled and not ch_k8s_lbaas_enable_snat + + - name: Remove no-SNAT configuration + file: + state: absent + path: "/etc/bird.d/01-no-snat.conf" + notify: restart bird + when: openstack_enabled and ipv4_enabled and ch_k8s_lbaas_enable_snat + - name: Start service systemd: name: ch-k8s-lbaas-agent @@ -93,6 +120,7 @@ group: root mode: "u=rw,go=r" notify: reload nftables + when: openstack_enabled - name: Disable LBaaS agent when: not ch_k8s_lbaas_enabled diff --git a/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/templates/agent-config.toml.j2 b/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/templates/agent-config.toml.j2 index 174d0f4ede094c86da21a820751db24f5f22cfdf..4f373bda901902debf96bb1a6751ff55ce32c56c 100644 --- a/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/templates/agent-config.toml.j2 +++ b/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/templates/agent-config.toml.j2 @@ -13,5 +13,8 @@ virtual-router-id-base=10 config-file="/var/lib/ch-k8s-lbaas-agent/keepalived/lbaas.conf" check-delay=2 +[nftables] +enable-snat={{ ch_k8s_lbaas_enable_snat | bool | lower }} + [nftables.service] config-file="/var/lib/ch-k8s-lbaas-agent/nftables/lbaas.conf" diff --git a/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/templates/bird.no-snat.conf.j2 b/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/templates/bird.no-snat.conf.j2 new file mode 100644 index 0000000000000000000000000000000000000000..70747673425a08a981b3cf05bf97e8b1d953a7db --- /dev/null +++ b/k8s-supplements/ansible/roles/ch-k8s-lbaas-agent/templates/bird.no-snat.conf.j2 @@ -0,0 +1,16 @@ +protocol static { + table master; + # Add routes to basically route everything via the VIP + route 0.0.0.0/1 via {{ networking_fixed_ip }}; + route 128.0.0.0/1 via {{ networking_fixed_ip }}; +} + +# A function used to export the above routes to peers +function filter_default_overwrite() { + if (net = 0.0.0.0/1) || (net = 128.0.0.0/1) then { + bgp_local_pref = 200; # Set local preference to override default route on peers + bgp_next_hop = {{ networking_fixed_ip }}; + return true; + } + return false; +} diff --git a/k8s-supplements/ansible/roles/nftables/templates/nftables.conf.j2 b/k8s-supplements/ansible/roles/nftables/templates/nftables.conf.j2 index 5085e3995f33aa99cc4a031b6d1eb4253e5c6191..dbad0abaa1e1aacc3e732ef031d12bd5777ff7df 100644 --- a/k8s-supplements/ansible/roles/nftables/templates/nftables.conf.j2 +++ b/k8s-supplements/ansible/roles/nftables/templates/nftables.conf.j2 @@ -167,6 +167,10 @@ include "/var/lib/ch-k8s-lbaas-agent/nftables/*.conf" table ip nat { chain postrouting { - fib saddr type != local masquerade; +{% if ch_k8s_lbaas_enabled and not ch_k8s_lbaas_enable_snat %} + ip saddr {{ priv_ip_cidr }} masquerade comment "Only SNAT internal traffic"; +{% else %} + fib saddr type != local masquerade comment "SNAT all traffic"; +{% endif %} } } diff --git a/k8s-supplements/ansible/roles/wireguard/templates/wg_bird.conf.j2 b/k8s-supplements/ansible/roles/wireguard/templates/wg_bird.conf.j2 index c1f0be2c1c45e306e35f2d99abb1884b4f61a88b..4194209f180849df8bdef2d0dc78dabc2e7dba71 100644 --- a/k8s-supplements/ansible/roles/wireguard/templates/wg_bird.conf.j2 +++ b/k8s-supplements/ansible/roles/wireguard/templates/wg_bird.conf.j2 @@ -1,11 +1,10 @@ {{ _auto_generated_preamble }} -filter k8s_worker { - {% for ep in wg_endpoints %} - {% if ep.enabled and ep.ip_cidr is not none %} - if net = {{ ep.ip_cidr }} then accept; - {% endif %} - {% endfor %} - if scope = SCOPE_ORGANIZATION then accept; - else reject; +function filter_wireguard() { +{% for ep in wg_endpoints %} +{% if ep.enabled and ep.ip_cidr is not none %} + if net = {{ ep.ip_cidr }} then return true; +{% endif %} +{% endfor %} + return false; } diff --git a/k8s-supplements/ansible/roles/wireguard/templates/wg_bird_v6.conf.j2 b/k8s-supplements/ansible/roles/wireguard/templates/wg_bird_v6.conf.j2 index 7bd24bac51f82e713d04321afe0b60f20e3d6598..f471b211ac626133fbf1502e8830d497f38278b9 100644 --- a/k8s-supplements/ansible/roles/wireguard/templates/wg_bird_v6.conf.j2 +++ b/k8s-supplements/ansible/roles/wireguard/templates/wg_bird_v6.conf.j2 @@ -1,11 +1,10 @@ {{ _auto_generated_preamble }} -filter k8s_worker_v6 { - {% for ep in wg_endpoints %} - {% if ep.enabled and ep.ipv6_cidr is not none %} - if net = {{ ep.ipv6_cidr }} then accept; - {% endif %} - {% endfor %} - if scope = SCOPE_ORGANIZATION then accept; - else reject; +function filter_wireguard_v6() { +{% for ep in wg_endpoints %} +{% if ep.enabled and ep.ipv6_cidr is not none %} + if net = {{ ep.ipv6_cidr }} then return true; +{% endif %} +{% endfor %} + return false; } diff --git a/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix b/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix index e244de4d6d9817c9e7c7694d767f44a3089bac71..5b28793e7eeb7a802c50cbe0930548885e3d2abf 100644 --- a/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix +++ b/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix @@ -124,6 +124,40 @@ in { type = types.bool; default = config.yk8s.kubernetes.network.calico.enabled; }; + enable_snat = mkDisableOption '' + source-nat'ing by the ch-k8s-lbaas-agents running on the frontend nodes. + + Disabling this has a similar effect as a direct server return. + It allows to see the real source IP of traffic sent to a LoadBalancer-service. + + After reconfiguring this option, execute the following: + + .. code:: + + $ ./managed-k8s/actions/apply-k8s-supplements.sh install-ch-k8s-lbaas.yaml + + to rollout necessary changes. + + Running on OpenStack + """""""""""""""""""" + + If source-nat'ing is disabled, the frontend nodes will be configured to act as gateway + for the Kubernetes nodes. They will propagate routes via BGP overwriting the default routes of + Kubernetes nodes such that **all** traffic is routed via the VIP by default. + + .. warning:: Implications when running on OpenStack + + Disabling source-nat'ing has some implications: + + 1. If a failover occurs on the frontend nodes, **all** connections are impacted, + not only connections to LoadBalancer-Services. + 2. The source IP of Kubernetes nodes as seen by the outside world changes from + the OpenStack router IP to the Gateway's VIP. + 3. It's not possible to attach floating IPs to Kubernetes nodes anymore due to + routing asymmetry. + + Be aware, that the frontend nodes must be potent enough to handle the increased amount of traffic + if source-nat'ing is disabled, as they could become the bottleneck otherwise''; }; config.yk8s.assertions = [ # Due to OVN support, require version >= 0.8.0 (warn only if not in semver2 format)