diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 319cd45ed05da9b9a1a029f9c7a1f41879e1629f..5d10ea33b11a6b2a6e47ed679c0cb00aa298f0db 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -61,23 +61,6 @@ include: - if: $CI_MERGE_REQUEST_EVENT_TYPE == "merge_train" when: on_success -tflint: - image: - name: "ghcr.io/terraform-linters/tflint:v0.59.1" - entrypoint: ["/bin/sh", "-c"] - stage: lint - script: - - tflint --chdir=terraform/ - tags: - - docker - rules: - - !reference [.default_lint_rules, rules] - - changes: - - 'terraform/**/*' - if: $SKIP_LINT != "true" - when: on_success - - when: never - shellcheck: image: "${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/koalaman/shellcheck-alpine:v0.11.0" stage: lint diff --git a/actions/apply-all.sh b/actions/apply-all.sh index 9f5fbc5c75bd73dd44d404047d8f558765351f36..536c409214ef8ed16c00d2ea7d9a6e6b09256a0e 100755 --- a/actions/apply-all.sh +++ b/actions/apply-all.sh @@ -6,7 +6,7 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars diff --git a/actions/apply-custom.sh b/actions/apply-custom.sh index f26cff0d9219d2d26cedbd8c1a647c9aea07b306..a3ec051e1e6fc72510e5057d02a7f18a80c34d0e 100755 --- a/actions/apply-custom.sh +++ b/actions/apply-custom.sh @@ -6,14 +6,12 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars check_venv -check_conf_sanity - require_vault_token install_prerequisites @@ -27,6 +25,8 @@ set_kubeconfig ANSIBLE_ROLES_PATH="$ansible_k8s_core_dir/roles:$ansible_k8s_supplements_dir/roles:$ansible_k8s_custom_playbook_dir/roles" export ANSIBLE_ROLES_PATH +"$actions_dir/update-inventory.sh" ansible + pushd "$ansible_k8s_custom_dispatch_dir" ansible_playbook \ -i "$ansible_inventory_host_file" \ diff --git a/actions/apply-k8s-core.sh b/actions/apply-k8s-core.sh index 22a180796e43a6cd03551184ffc7a80360b35418..ea511cb0e4e8a52e5d240addbfcadfb4844dd2b2 100755 --- a/actions/apply-k8s-core.sh +++ b/actions/apply-k8s-core.sh @@ -26,17 +26,18 @@ execute_playbook() { notef "Executing playbook $playbook\n" # Ensure that the latest config is deployed to the inventory - "$actions_dir/update-inventory.sh" + "$actions_dir/update-inventory.sh" conf_vars load_conf_vars check_venv - check_conf_sanity require_vault_token install_prerequisites # Bring the wireguard interface up if configured so "$actions_dir/wg-up.sh" + "$actions_dir/update-inventory.sh" ansible + pushd "$ansible_k8s_core_dir" # Include k8s-core roles ANSIBLE_ROLES_PATH="$ansible_k8s_core_dir/roles" \ diff --git a/actions/apply-k8s-supplements.sh b/actions/apply-k8s-supplements.sh index 0c9ba84f2e79e9d861527d5b4ea4985a24b758fb..7b7d7242da341394752ea276f016bb20b7b469c9 100755 --- a/actions/apply-k8s-supplements.sh +++ b/actions/apply-k8s-supplements.sh @@ -26,10 +26,9 @@ execute_playbook() { notef "Executing playbook $playbook\n" # Ensure that the latest config is deployed to the inventory - "$actions_dir/update-inventory.sh" + "$actions_dir/update-inventory.sh" conf_vars load_conf_vars - check_conf_sanity check_venv require_vault_token install_prerequisites @@ -39,6 +38,8 @@ execute_playbook() { set_kubeconfig + "$actions_dir/update-inventory.sh" ansible + pushd "$ansible_k8s_supplements_dir" # Include k8s-core roles ANSIBLE_ROLES_PATH="$ansible_k8s_core_dir/roles:$ansible_k8s_supplements_dir/roles" \ diff --git a/actions/apply-prepare-gw.sh b/actions/apply-prepare-gw.sh index 10ed7ec119cd950d5cb45e69bf6acbadfab5b62b..a8f4aa32a0bdb102fe67172e98d117da1ed05d6a 100755 --- a/actions/apply-prepare-gw.sh +++ b/actions/apply-prepare-gw.sh @@ -6,13 +6,12 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars check_venv -check_conf_sanity require_vault_token @@ -26,6 +25,8 @@ if [ "${tf_usage:-true}" == 'false' ]; then exit 1 fi +"$actions_dir/update-inventory.sh" ansible + # Prepare Gateways, if configured pushd "$ansible_k8s_supplements_dir" # Include k8s-core common roles diff --git a/actions/apply-terraform.sh b/actions/apply-terraform.sh index a0fb1a0a1e0d30e231e119c71421994f05f4c0a8..1ce607405537889cbeb0aec001bad7e4c2fc969d 100755 --- a/actions/apply-terraform.sh +++ b/actions/apply-terraform.sh @@ -6,7 +6,7 @@ actions_dir="$(realpath "$(dirname "$0")")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars @@ -17,15 +17,16 @@ if [ "$("$actions_dir/helpers/semver2.sh" "$(terraform -v -json | jq -r '.terraf exit 5 fi -var_file="$terraform_state_dir/config.tfvars.json" +"$actions_dir/update-inventory.sh" terraform + cd "$terraform_state_dir" tf_init -run terraform -chdir="$terraform_module" plan --var-file="$var_file" --out "$terraform_plan" +run terraform plan --out "$terraform_plan" # strict mode terminates the execution of this script immediately set +e -terraform -chdir="$terraform_module" show -json "$terraform_plan" | python3 "$actions_dir/helpers/check_plan.py" +terraform show -json "$terraform_plan" | python3 "$actions_dir/helpers/check_plan.py" rc=$? set -e RC_DISRUPTION=47 @@ -39,7 +40,8 @@ elif [ $rc != $RC_NO_DISRUPTION ] && [ $rc != $RC_DISRUPTION ]; then errorf 'error during execution of check_plan.py. Aborting' >&2 exit 4 fi -run terraform -chdir="$terraform_module" apply "$terraform_plan" +run terraform apply "$terraform_plan" | sed '/Outputs:/,$d' # we don't want Terraform to dump all the outputs to the console +terraform output -json > "$terraform_state_dir/outputs.json" if [ "$(jq -r .backend.type "$terraform_state_dir/.terraform/terraform.tfstate")" == 'http' ]; then notef 'Pulling latest Terraform state from Gitlab for disaster recovery purposes.' diff --git a/actions/destroy.sh b/actions/destroy.sh index 5a16b96fa14d493dbe6d568febe540fbde0f8bfa..5f314d5d11833bb35f5e80e09585a92dfa83114f 100755 --- a/actions/destroy.sh +++ b/actions/destroy.sh @@ -6,7 +6,7 @@ actions_dir="$(realpath "$(dirname "$0")")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars @@ -66,11 +66,13 @@ if [ "${#port_ids[@]}" != 0 ]; then run openstack port delete "${port_ids[@]}" fi +"$actions_dir/update-inventory.sh" terraform + cd "$terraform_state_dir" export TF_DATA_DIR="$terraform_state_dir/.terraform" -run terraform -chdir="$terraform_module" init +run terraform init # The following task will fail if a) thanos wrote data into a container and b) `MANAGED_K8S_NUKE_FROM_ORBIT` is not set -run terraform -chdir="$terraform_module" destroy --var-file="$terraform_state_dir/config.tfvars.json" --auto-approve || true +run terraform destroy --auto-approve || true IFS=$'\n' read -r -d '' -a volume_ids < <( openstack volume list --project "$OS_PROJECT_ID" -f value -c ID && printf '\0' ) if [ "${#volume_ids[@]}" != 0 ]; then @@ -93,7 +95,6 @@ rm -f inventory/.etc/wg_gw_pub.key if [ "$(jq -r .backend.type "$terraform_state_dir/.terraform/terraform.tfstate")" == 'http' ] ; then GITLAB_RESPONSE=$(curl -Is --header "Private-Token: $TF_HTTP_PASSWORD" -o "/dev/null" -w "%{http_code}" --request DELETE "$backend_address") check_return_code "$GITLAB_RESPONSE" - rm -f "$terraform_module/backend_override.tf" fi # Purge the remaining terraform directory. Its existence is a condition for additional disruption checks. diff --git a/actions/k8s-login.sh b/actions/k8s-login.sh index 52e32892a696b80c48863d2bc160936b682419fb..6fee679bc91cc7391dcfd266c4d3f63e7d7ced85 100755 --- a/actions/k8s-login.sh +++ b/actions/k8s-login.sh @@ -6,11 +6,10 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars -check_conf_sanity require_vault_token @@ -41,6 +40,8 @@ fi check_vault_token_policy +"$actions_dir/update-inventory.sh" ansible + pushd "$ansible_k8s_core_dir" # Include k8s-core roles ANSIBLE_ROLES_PATH="$ansible_k8s_core_dir/roles" \ diff --git a/actions/lib.sh b/actions/lib.sh index b3b87691e6a32eaad1ee0b44eb739a9e18cae4d1..bed982bd1c9a9a829bc4bb664688f6bed5f42017 100644 --- a/actions/lib.sh +++ b/actions/lib.sh @@ -4,6 +4,7 @@ cluster_repository="$(realpath ".")" code_repository="$(realpath "$actions_dir/../")" etc_directory="$(realpath "etc")" group_vars_dir="${cluster_repository}/inventory/yaook-k8s/group_vars" +conf_vars_file="${cluster_repository}/inventory/conf_vars/main.yaml" state_dir="$cluster_repository/state" release_migration_lock="$state_dir/release-migration-in-progress" @@ -16,7 +17,6 @@ terraform_min_version="1.3.0" terraform_state_dir="$state_dir/terraform" terraform_disruption_lock="$terraform_state_dir/prevent_disruption.lock" export TF_DATA_DIR="$terraform_state_dir/.terraform" -terraform_module="${TERRAFORM_MODULE_PATH:-$code_repository/terraform}" terraform_plan="$terraform_state_dir/plan.tfplan" ansible_directory="$code_repository/ansible" @@ -72,23 +72,16 @@ function load_conf_vars() { # All the things with side-effects should go here terraform_prevent_disruption="$(if [ -e "$terraform_disruption_lock" ]; then echo "true"; else echo "false"; fi)" - tf_usage=${tf_usage:-"$(yq '. | if has ("enabled") then .enabled else true end' "$group_vars_dir/all/terraform.yaml")"} - wg_usage=${wg_usage:-"$(yq '. | if has("enabled") then .enabled else true end' "$group_vars_dir/gateways/wireguard.yaml")"} + tf_usage="$(yq '.tf_usage' "$conf_vars_file")" + wg_usage="$(yq '.wg_usage' "$conf_vars_file")" if [ "${wg_usage:-true}" == "true" ]; then wg_conf="${wg_conf:-$cluster_repository/${wg_conf_name}.conf}" wg_interface="$(basename "$wg_conf" | cut -d'.' -f1)" wg_endpoint="${wg_endpoint:-0}" ansible_wg_template="$etc_directory/wireguard/wg${wg_endpoint}/wg${wg_endpoint}_${wg_user}.conf" - fi -} - -function check_conf_sanity() { - out=$(ansible-inventory -i "${ansible_inventory_base}" --host localhost) - if ! (jq --exit-status '.ipv4_enabled or .ipv6_enabled' <<<"${out}" &> /dev/null); then - errorf "Neither IPv4 nor IPv6 are enabled." - errorf "Enable at least one in your hosts file $ansible_inventory_host_file." - exit 2 + wg_subnet="$(yq -r .wg_subnet "$conf_vars_file")" + wg_subnet_v6="$(yq -r .wg_subnet_v6 "$conf_vars_file")" fi } @@ -228,10 +221,10 @@ function ansible_playbook() { } function load_gitlab_vars() { - gitlab_base_url="$(jq -r .gitlab_base_url "$terraform_state_dir/config.tfvars.json")" - gitlab_project_id="$(jq -r .gitlab_project_id "$terraform_state_dir/config.tfvars.json")" - gitlab_state_name="$(jq -r .gitlab_state_name "$terraform_state_dir/config.tfvars.json")" - backend_address="$gitlab_base_url/api/v4/projects/$gitlab_project_id/terraform/state/$gitlab_state_name" + gitlab_base_url="$(jq -r .gitlab_base_url "$terraform_state_dir/gitlab.conf.json")" + gitlab_project_id="$(jq -r .gitlab_project_id "$terraform_state_dir/gitlab.conf.json")" + gitlab_state_name="$(jq -r .gitlab_state_name "$terraform_state_dir/gitlab.conf.json")" + backend_address="$(jq -r .gitlab_backend_address "$terraform_state_dir/gitlab.conf.json")" } # true: HTTP/200 response; false: HTTP/404; exit: HTTP/* @@ -275,39 +268,14 @@ function check_venv() { } function tf_init() { - OVERRIDE_FILE="$terraform_module/backend_override.tf" - - function tf_init_http () { - run terraform -chdir="$terraform_module" init \ - -upgrade \ - -backend-config="address=$backend_address" \ - -backend-config="lock_address=$backend_address/lock" \ - -backend-config="unlock_address=$backend_address/lock" \ - -backend-config="lock_method=POST" \ - -backend-config="unlock_method=DELETE" \ - -backend-config="retry_wait_min=5" - } - function tf_init_local () { - run terraform -chdir="$terraform_module" init \ + function _tf_init () { + run terraform init \ -upgrade } - function tf_init_http_migrate () { - run terraform -chdir="$terraform_module" init \ - -migrate-state \ - -force-copy \ - -upgrade \ - -backend-config="address=$backend_address" \ - -backend-config="lock_address=$backend_address/lock" \ - -backend-config="unlock_address=$backend_address/lock" \ - -backend-config="lock_method=POST" \ - -backend-config="unlock_method=DELETE" \ - -backend-config="retry_wait_min=5" - } - - function tf_init_local_migrate () { - run terraform -chdir="$terraform_module" init \ + function _tf_init_migrate () { + run terraform init \ -migrate-state \ -force-copy \ -upgrade @@ -323,13 +291,6 @@ function tf_init() { return 0 } - function all_gitlab_vars_are_unset() { - for var in "${all_gitlab_vars[@]}"; do - [[ -n "${!var}" && "${!var}" != "null" ]] && return 1 - done - return 0 - } - function tf_state_present_on_gitlab () { if [ -z "${TF_HTTP_PASSWORD:-}" ]; then errorf "We want to check if there is a Gitlab state present," @@ -355,52 +316,26 @@ function tf_init() { fi # gitlab_backend=true - if [ "$(jq -r .gitlab_backend "$terraform_state_dir/config.tfvars.json")" = true ]; then - if ! all_gitlab_vars_are_set; then - errorf "'gitlab_backend=true' but GitLab variables are not (completely) provided." - exit 2 - fi - - # Here we create an override_file which overrides the `local` terraform backend to http(gitlab) backend - if [ ! -f "$OVERRIDE_FILE" ]; then - cat > "$OVERRIDE_FILE" <<-EOF -terraform { - backend "http" {} -} -EOF - fi - + if [ "$(jq -r '.terraform.backend | has("http")' "$terraform_state_dir/config.tf.json")" = true ]; then if tf_state_present_on_gitlab; then - tf_init_http + _tf_init else if [ -f "$terraform_state_dir/terraform.tfstate" ]; then - tf_init_http_migrate + _tf_init_migrate # Delete terraform statefiles locally if they exist (-f) rm -f "$terraform_state_dir/terraform.tfstate" "$terraform_state_dir/terraform.tfstate.backup" else - tf_init_http # first init + _tf_init # first init fi fi # gitlab_backend=false else - if ! all_gitlab_vars_are_set && ! all_gitlab_vars_are_unset; then - errorf "'gitlab_backend=false' but some GitLab variables are provided." - errorf "(1) If you want to migrate the Terraform backend method from 'http' to 'local'," - errorf "you should provide all the GitLab variables" - errorf "(2) If you want to init a cluster with local backend," - errorf "make sure that all the following GitLab variables are unset:" - for var in "${all_gitlab_vars[@]}"; do - errorf "- $var" - done - exit 2 - fi if all_gitlab_vars_are_set; then if tf_state_present_on_gitlab; then - rm -f "$OVERRIDE_FILE" notef "Terraform statefile on GitLab found. Migration from http to local." - if tf_init_local_migrate; then + if _tf_init_migrate; then # delete tf_statefile from GitLab GITLAB_RESPONSE=$(curl -Is --header "Private-Token: $TF_HTTP_PASSWORD" -o "/dev/null" -w "%{http_code}" --request DELETE "$backend_address") check_return_code "$GITLAB_RESPONSE" @@ -419,8 +354,7 @@ EOF exit 2 fi else - rm -f "$OVERRIDE_FILE" - tf_init_local + _tf_init fi fi } diff --git a/actions/manual-terraform.sh b/actions/manual-terraform.sh index 7a525790359f0cc1c552847765cca9d2839c2459..4f649ab05a490f3fbc5a0bc988a5ac6e292c613c 100755 --- a/actions/manual-terraform.sh +++ b/actions/manual-terraform.sh @@ -6,8 +6,8 @@ actions_dir="$(realpath "$(dirname "$0")")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" terraform cd "$terraform_state_dir" export TF_DATA_DIR="$terraform_state_dir/.terraform" -exec terraform -chdir="$terraform_module" "$@" +exec terraform "$@" diff --git a/actions/release-migrations/a-10-terraform-state-migrations.sh b/actions/release-migrations/a-10-terraform-state-migrations.sh new file mode 100755 index 0000000000000000000000000000000000000000..d3a98fc9cac4ebd7a9a9ba9b7bf545be5b32fccc --- /dev/null +++ b/actions/release-migrations/a-10-terraform-state-migrations.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash + +set -euo pipefail +actions_dir="$(dirname "$0")/.." + +# shellcheck source=actions/lib.sh +. "$actions_dir/lib.sh" + +# Ensure that the latest config is deployed to the inventory +"$actions_dir/update-inventory.sh" conf_vars + +load_conf_vars + +if [ "${tf_usage:-true}" != 'true' ]; then + notef "Skipped Terraform related migration because it is disabled." + exit 0 +fi + +"$actions_dir/update-inventory.sh" terraform + +pushd "$terraform_state_dir" + +tf_init + +notef "Checking if Terraform migrations exist..." +migrations=$(nix build --override-input yk8s "$code_repository" --print-out-paths --no-link --show-trace .#tf-state-migrations) +if ! [ -s "$migrations" ]; then + notef "Nothing to do." + exit 0 +fi + +notef "Checking if Terraform migration is necessary..." +run terraform plan --out "$terraform_plan" +# strict mode terminates the execution of this script immediately +set +e +terraform show -json "$terraform_plan" | python3 "$actions_dir/helpers/check_plan.py" +rc=$? +set -e +RC_DISRUPTION=47 +RC_NO_DISRUPTION=0 +if [ $rc == $RC_NO_DISRUPTION ]; then + notef "Nothing to do." + exit 0 +elif [ $rc != $RC_NO_DISRUPTION ] && [ $rc != $RC_DISRUPTION ]; then + errorf 'error during execution of check_plan.py. Aborting' >&2 + exit 4 +fi + +notef "Migrating Terraform state..." + +tf_statefile_temp="$terraform_state_dir/terraform.tfstate.tmp" +terraform state pull > "$tf_statefile_temp" + +# shellcheck disable=SC1090 +source "$migrations" + +run terraform state push "$tf_statefile_temp" + +notef "Terraform state migration done." +notef "Running apply-terraform to create their replacements..." + +popd + +run "$actions_dir/apply-terraform.sh" + + +rm "$tf_statefile_temp" + +run git add "$terraform_state_dir" diff --git a/actions/release-migrations/v10-00-01-terraform.sh b/actions/release-migrations/v10-00-01-terraform.sh deleted file mode 100755 index ed7f3f73030e2ef1993b20cf0ea4876989ee0e08..0000000000000000000000000000000000000000 --- a/actions/release-migrations/v10-00-01-terraform.sh +++ /dev/null @@ -1,29 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -actions_dir="$(dirname "$0")/.." - -# shellcheck source=actions/lib.sh -. "$actions_dir/lib.sh" - -load_conf_vars - -if [ "${tf_usage:-true}" == 'true' ]; then - notef "Temporarily patching the Terraform module \ -to unignore allowed_address_pairs..." - if git -C "${code_repository}" apply --quiet --check --reverse \ -"$(realpath --no-symlinks --relative-to="${code_repository}" "$actions_dir")\ -/release-migrations/v10-00-tf-unignore-allowed_address_pairs.patch" - then - notef "Terraform module already got patched" - else - git -C "${code_repository}" apply \ -"$(realpath --no-symlinks --relative-to="${code_repository}" "$actions_dir")\ -/release-migrations/v10-00-tf-unignore-allowed_address_pairs.patch" - notef "Terraform module patched" - fi - - notef "Triggering Terraform (1/2)" - run "$actions_dir/apply-terraform.sh" - run "$actions_dir/update-inventory.sh" -fi diff --git a/actions/release-migrations/v10-00-02-ch-k8s-lbaas.sh b/actions/release-migrations/v10-00-02-ch-k8s-lbaas.sh deleted file mode 100755 index 8226d7cec86cae5b4e930ee03504a387f0c7a31b..0000000000000000000000000000000000000000 --- a/actions/release-migrations/v10-00-02-ch-k8s-lbaas.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/env bash - -# NOTE: This migration must be run before port security is enabled by Terraform - -set -euo pipefail -actions_dir="$(dirname "$0")/.." - -# shellcheck source=actions/lib.sh -. "$actions_dir/lib.sh" - -# Let ch-k8s-lbaas version >= 0.8.0 be enforced by Nix -run "$actions_dir/update-inventory.sh" - -notef "Triggering install-ch-k8s-lbaas.yaml playbook to update ch-k8s-lbaas." -run "$actions_dir/apply-k8s-supplements.sh" install-ch-k8s-lbaas.yaml diff --git a/actions/release-migrations/v10-00-03-terraform.sh b/actions/release-migrations/v10-00-03-terraform.sh deleted file mode 100755 index 5d1cd05dea06de13939190b28fa7f508cdaec82e..0000000000000000000000000000000000000000 --- a/actions/release-migrations/v10-00-03-terraform.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash - -set -euo pipefail -actions_dir="$(dirname "$0")/.." - -# shellcheck source=actions/lib.sh -. "$actions_dir/lib.sh" - -load_conf_vars - -if [ "${tf_usage:-true}" == 'true' ]; then - notef "Reverting temporarily patched Terraform module..." - git -C "${code_repository}" apply --reverse \ - "$(realpath --no-symlinks --relative-to="${code_repository}" "$actions_dir")\ -/release-migrations/v10-00-tf-unignore-allowed_address_pairs.patch" - - notef "Triggering Terraform (2/2)" - run "$actions_dir/apply-terraform.sh" - run "$actions_dir/update-inventory.sh" - - # After enabling port security with Terraform - # the cluster nodes will be unreachable for a short period of time. - # Migration can only continue afterwards. - notef "Checking that the Kubernetes API is reachable (timeout=4s)..." - until (kubectl --request-timeout=4s api-versions &>/dev/null); do - echo "Failed, retrying in 4s..." - sleep 4 - done - echo "Kubernetes API is reachable. Continuing..." -fi diff --git a/actions/release-migrations/v10-00-tf-unignore-allowed_address_pairs.patch b/actions/release-migrations/v10-00-tf-unignore-allowed_address_pairs.patch deleted file mode 100644 index 46f78a6fc725591f6458541f8830298d738946e4..0000000000000000000000000000000000000000 --- a/actions/release-migrations/v10-00-tf-unignore-allowed_address_pairs.patch +++ /dev/null @@ -1,18 +0,0 @@ -diff --git i/terraform/20-gateway.tf w/terraform/20-gateway.tf -index 27e5a57a6..f0e5051e6 100644 ---- i/terraform/20-gateway.tf -+++ w/terraform/20-gateway.tf -@@ -114,13 +114,6 @@ resource "openstack_networking_port_v2" "gateway" { - - lifecycle { - ignore_changes = [ -- # The allowed_address_pairs are subject to change and may get -- # (automatically) managed or extended by something else. -- # For example, the ch-k8s-lbaas controller manages them to -- # allow LBaaS traffic. -- # Terraform would reset these settings on each run if we would -- # not ignore changes -- allowed_address_pairs - ] - } - } diff --git a/actions/release-migrations/v10-01-direnv-layout.sh b/actions/release-migrations/v10-01-direnv-layout.sh deleted file mode 100755 index 14e681c8b59b47b87809b6b14962cda6a1b4e217..0000000000000000000000000000000000000000 --- a/actions/release-migrations/v10-01-direnv-layout.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -actions_dir="$(dirname "$0")/.." - -# shellcheck source=actions/lib.sh -. "$actions_dir/lib.sh" - -pushd "$cluster_repository" >/dev/null || exit 1 - -notef "Updating .envrc" -run sed -i 's#use flake_if_nix ./managed-k8s#layout yaook-k8s#g' .envrc -run sed -i '\#layout poetry#d' .envrc -run sed -i '\#source_env ~/.config/yaook-k8s/env || true#d' .envrc -run sed -i '\#source_env_if_exists ~/.config/yaook-k8s/env#d' .envrc -# shellcheck disable=SC2016 -run sed -i '\#source_env "$PWD/.envrc.local" || true#d' .envrc -# shellcheck disable=SC2016 -run sed -i '\#source_env_if_exists $PWD/.envrc.local#d' .envrc -run sed -i 's#\(source_env ./managed-k8s/.envrc.lib.sh\) || true#\1#' .envrc - -run git add .envrc - -popd >/dev/null || exit 1 diff --git a/actions/release-migrations/v10-02-keepalived.sh b/actions/release-migrations/v10-02-keepalived.sh deleted file mode 100755 index 4c1b24138bfe705181e3640b403b9ab841d6c8aa..0000000000000000000000000000000000000000 --- a/actions/release-migrations/v10-02-keepalived.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -# We must update keepalived configuration - -set -euo pipefail -actions_dir="$(dirname "$0")/.." - -# shellcheck source=actions/lib.sh -. "$actions_dir/lib.sh" - -notef "Updating keepalived configuration." -AFLAGS="--diff -t keepalived" run "$actions_dir/apply-k8s-core.sh" install-frontend-services.yaml diff --git a/actions/release-migrations/v10-02-prevent-disruption.sh b/actions/release-migrations/v10-02-prevent-disruption.sh deleted file mode 100644 index 555d8365e05a0ad49dd8fc85e3e509ca5c716d17..0000000000000000000000000000000000000000 --- a/actions/release-migrations/v10-02-prevent-disruption.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -actions_dir="$(dirname "$0")/.." - -# shellcheck source=actions/lib.sh -. "$actions_dir/lib.sh" - -pushd "$cluster_repository" >/dev/null || exit 1 - -notef "Creating Terraform disruption lock" -touch "$terraform_disruption_lock" -git add "$terraform_disruption_lock" - -popd >/dev/null || exit 1 diff --git a/actions/rotate-root-ca.sh b/actions/rotate-root-ca.sh index 6786e9e783e43e0baa334ef25c6f6157934123de..fc0ab0440e01b6df2953207e3f51783b12fee70c 100755 --- a/actions/rotate-root-ca.sh +++ b/actions/rotate-root-ca.sh @@ -43,13 +43,12 @@ fi . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars check_venv -check_conf_sanity require_ansible_disruption @@ -69,6 +68,8 @@ if [ "${complete_rotation:-false}" == "false" ]; then run "$actions_dir/k8s-login.sh" fi +"$actions_dir/update-inventory.sh" ansible + pushd "$ansible_k8s_core_dir" ansible_playbook -i "$ansible_inventory_host_file" \ -e "append_next_issuer=${next_issuer:-false}" \ diff --git a/actions/test.sh b/actions/test.sh index 70bb3c5d1b2d7478804f70620e8e5698b04d9f75..e559d0f6df14f79fa455e2e372537b8078aaad5e 100755 --- a/actions/test.sh +++ b/actions/test.sh @@ -7,11 +7,10 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars -check_conf_sanity check_venv @@ -20,6 +19,8 @@ check_venv set_kubeconfig +"$actions_dir/update-inventory.sh" ansible + # Test all pushd "$ansible_k8s_supplements_dir" ANSIBLE_ROLES_PATH="$ansible_k8s_core_dir/roles/:$ansible_k8s_supplements_dir/test-roles:$ansible_k8s_supplements_dir/roles/" \ diff --git a/actions/update-frontend-nodes.sh b/actions/update-frontend-nodes.sh index f6c98df727a44fa21d204127eed37406f027f26d..97d6b05b2e97b0dc0eb0084c20e6889efd980edf 100755 --- a/actions/update-frontend-nodes.sh +++ b/actions/update-frontend-nodes.sh @@ -6,11 +6,10 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars -check_conf_sanity check_venv @@ -38,6 +37,8 @@ install_prerequisites set_kubeconfig +"$actions_dir/update-inventory.sh" ansible + # Trigger whole LCM pushd "$ansible_k8s_core_dir" # Include k8s-core roles diff --git a/actions/update-inventory.sh b/actions/update-inventory.sh index d5c1fe6c733524bba1009e445a5881451a8df0bd..0d467af71480ec285588b01c692b68d18a47b18f 100755 --- a/actions/update-inventory.sh +++ b/actions/update-inventory.sh @@ -2,6 +2,26 @@ set -euo pipefail actions_dir="$(dirname "$0")" +function usage() { + echo "usage: $0 target" >&2 + echo >&2 + echo "Arguments:" >&2 + echo " target" >&2 + echo " Possible values include: conf_vars, vault, terraform, ansible" >&2 +} + +arg_num=1 +if [ "$#" -lt "$arg_num" ]; then + echo "ERROR: Expecting $arg_num argument(s), but $# were given" >&2 + echo + usage + echo >&2 + exit 2 +fi + +target="$1" +shift + # shellcheck source=actions/lib.sh . "$actions_dir/lib.sh" @@ -17,13 +37,18 @@ if [[ -e "inventory/yaook-k8s/hosts" ]] && [[ ! -L "inventory/yaook-k8s/hosts" ] fi if [[ -e "state" ]]; then git add state; fi if [ -z "${TAROOK_NIX_FLAGS:-}" ]; then - out=$(nix build --override-input yk8s "$code_repository" --print-out-paths --no-link "$@" .#yk8s-outputs) + out=$(nix build --override-input yk8s "$code_repository" --print-out-paths --no-link "$@" ".#yk8s-outputs-$target") else - out=$(nix build --override-input yk8s "$code_repository" --print-out-paths --no-link "${TAROOK_NIX_FLAGS}" "$@" .#yk8s-outputs) + out=$(nix build --override-input yk8s "$code_repository" --print-out-paths --no-link "${TAROOK_NIX_FLAGS}" "$@" ".#yk8s-outputs-$target") fi - -rsync -rL --chmod 664 "$out/state" . -rm -rf inventory -mkdir -p inventory/yaook-k8s/ -rsync -rl --chmod 664 "$out/inventory/yaook-k8s/" inventory/yaook-k8s/ +# shellcheck disable=SC1091 +. "$out/.path-info" +# shellcheck disable=SC2154 +rsync -rL --chmod 664 "$out/$state" . git add state +# shellcheck disable=SC2154 +if [ "$inventory" != "" ]; then + rm -rf "$inventory" + mkdir -p "$inventory" + rsync -rl --chmod 664 "$out/$inventory/" "$inventory" +fi diff --git a/actions/update-kubernetes-nodes.sh b/actions/update-kubernetes-nodes.sh index 89db811e61564d9c806cc79df7a8152ede6c8b8e..ae51e72d702c2311f1883f86707197ad2836b2ed 100755 --- a/actions/update-kubernetes-nodes.sh +++ b/actions/update-kubernetes-nodes.sh @@ -6,11 +6,10 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars -check_conf_sanity check_venv @@ -38,6 +37,8 @@ install_prerequisites set_kubeconfig +"$actions_dir/update-inventory.sh" ansible + pushd "$ansible_k8s_core_dir" # Include k8s-core roles ansible_playbook -i "$ansible_inventory_host_file" \ diff --git a/actions/upgrade.sh b/actions/upgrade.sh index 1e6897a4006359ff5e31126e139574e3c008d93d..7834a247d14f2569ee10e795680b7c32d9853b0b 100755 --- a/actions/upgrade.sh +++ b/actions/upgrade.sh @@ -6,11 +6,10 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars -check_conf_sanity check_venv @@ -65,6 +64,8 @@ require_ansible_disruption set_kubeconfig +"$actions_dir/update-inventory.sh" ansible + pushd "$ansible_k8s_supplements_dir" ANSIBLE_ROLES_PATH="$ansible_k8s_core_dir/roles:$ansible_k8s_supplements_dir/roles" \ ansible_playbook -i "$ansible_inventory_host_file" "$playbook" \ diff --git a/actions/verify-cluster-health.sh b/actions/verify-cluster-health.sh index c1c19f71e158cb255679d3f8c74013d73dd88181..913bbc2d23b144fb75648e1e204c1332c4171f38 100755 --- a/actions/verify-cluster-health.sh +++ b/actions/verify-cluster-health.sh @@ -6,11 +6,10 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars -check_conf_sanity while getopts s flag do @@ -31,6 +30,8 @@ check_venv set_kubeconfig +"$actions_dir/update-inventory.sh" ansible + pushd "$ansible_k8s_supplements_dir" # Include k8s-core roles ANSIBLE_ROLES_PATH="$ansible_k8s_core_dir/roles:$ansible_k8s_supplements_dir/roles" \ diff --git a/actions/wg-up.sh b/actions/wg-up.sh index a3529551a582a29a47c29b4f910f8a4390aa6544..6a63acd7fc81d167f8f17c500e3f450cf537402b 100755 --- a/actions/wg-up.sh +++ b/actions/wg-up.sh @@ -7,15 +7,13 @@ actions_dir="$(dirname "$0")" . "$actions_dir/lib.sh" # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" conf_vars load_conf_vars if [ "${wg_usage:-true}" == "true" ]; then validate_wireguard - wg_subnet="$(yq -r .subnet_cidr "$group_vars_dir/all/infra.yaml")" - wg_subnet_v6="$(yq -r .subnet_v6_cidr "$group_vars_dir/all/infra.yaml")" # the grep is there to ignore any routes going via the interface we're going to # take down later either way wg_existing_route="$(ip route show to "$wg_subnet" 2>/dev/null | grep -v "dev $wg_interface" || true)" diff --git a/docs/_releasenotes/1250.feature.1.yaml-hosts b/docs/_releasenotes/1250.feature.1.yaml-hosts new file mode 100644 index 0000000000000000000000000000000000000000..ae52d3dc9966ec731300ce4f600c2cf6848358a8 --- /dev/null +++ b/docs/_releasenotes/1250.feature.1.yaml-hosts @@ -0,0 +1 @@ +Ansible hosts can now be defined and referenced via Nix. See :ref:`configuration-options.yk8s.infra.ansible_hosts`. diff --git a/docs/_releasenotes/1250.removal.1.yaml-hosts b/docs/_releasenotes/1250.removal.1.yaml-hosts new file mode 100644 index 0000000000000000000000000000000000000000..6163cbc5cfc5a68d3c1d04155c0f80ad3a91c237 --- /dev/null +++ b/docs/_releasenotes/1250.removal.1.yaml-hosts @@ -0,0 +1 @@ +Importing an existing hosts file via :ref:`configuration-options.yk8s.infra.hosts_file` is deprecated. Hosts can be defined directly via Nix now. The option ``hosts_file`` will be removed at some point in the future. If you want to keep providing your own hosts file after that, convert it to YAML format (see https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html ) and import it like this ``ansible_hosts = yk8s-libs.importYAML ./hosts;``. diff --git a/docs/developer/guide/simulate-bm.rst b/docs/developer/guide/simulate-bm.rst index 090a4fc56c5c3fd82cd51c286298583f5b9b8633..ee665e02e301eb00f81f3e09fc85190f2fd2b3b0 100644 --- a/docs/developer/guide/simulate-bm.rst +++ b/docs/developer/guide/simulate-bm.rst @@ -87,23 +87,17 @@ their ports and associated floating IPs: openstack port delete "$gateway" done -Reconfigure the inventory ``inventory/yaook-k8s/hosts``: +Adapt the inventory and write it to ``config/hosts`` -.. code:: nix - - cp inventory/yaook-k8s/hosts config/hosts - chmod u+w config/hosts - -... and set ``infra.hosts_file = ./hosts;`` in the config. +.. code:: bash -Also remove the ``[gateways]`` section from ``config/hosts`` -and replace ``gateways`` with ``masters`` in the ``[frontend:children]`` section. + yq -y 'del(.gateways) | .frontend.children = {masters: {}}' inventory/yaook-k8s/hosts > config/hosts -We can now disable Terraform: +Now disable OpenStack and import our hosts file into the config: .. code:: nix - terraform.enable = false; + infra.ansible_hosts = yk8s-lib.importYAML ./hosts; openstack.enable = false; Create a jump host diff --git a/docs/user/reference/cluster-configuration.rst b/docs/user/reference/cluster-configuration.rst index 540557125817118d757636c0bbc8096771a8c692..3adf8d639ff4cbe09f421df67af5d52e6903a118 100644 --- a/docs/user/reference/cluster-configuration.rst +++ b/docs/user/reference/cluster-configuration.rst @@ -16,8 +16,7 @@ The cluster repository layout your_cluster_repo ├── config/ # All user configuration now resides in this directory - │ ├── default.nix # Nix-based cluster configuration - │ └── hosts # Manual Ansible hosts file for bare-metal, referenced in default.nix + │ └── default.nix # Nix-based cluster configuration ├── inventory/yaook-k8s/ # Ansible inventory is now completely generated and MAY be excluded from version control │ ├── group-vars/ # Variables passed to Ansible │ └── hosts # Ansible hosts file, generated from config even for bare-metal diff --git a/docs/user/reference/cluster-repository.rst b/docs/user/reference/cluster-repository.rst index 0712e12c9e9f1b298972c62b1af845ddbaab8796..20ed265b99e03f883f8ced3b044eaae33cc4e63f 100644 --- a/docs/user/reference/cluster-repository.rst +++ b/docs/user/reference/cluster-repository.rst @@ -19,7 +19,7 @@ the Tarook cluster. Cluster Repository Structure ---------------------------- -The following schema shows all non-generated files. A local checkout +The following schema shows an example set of files. A local checkout will most certainly have more files than these. :: diff --git a/docs/user/reference/options/yk8s.openstack.rst b/docs/user/reference/options/yk8s.openstack.rst index b70e689f7649f0374c06fd35db1c996efd785741..8cfcce2525ba90e888051fb16a4fbfe761264b9d 100644 --- a/docs/user/reference/options/yk8s.openstack.rst +++ b/docs/user/reference/options/yk8s.openstack.rst @@ -6,16 +6,16 @@ yk8s.openstack .. note:: - :ref:`configuration-options.yk8s.openstack.nodes` - allows you to configure - the k8s master and worker servers. - The ``role`` attribute must be used to distinguish both [1]_. + :ref:`configuration-options.yk8s.openstack.nodes` + allows you to configure + the k8s master and worker servers. + The ``role`` attribute must be used to distinguish both [1]_. - The amount of gateway nodes can be controlled with - :ref:`configuration-options.yk8s.openstack.gateway_count`. + The amount of gateway nodes can be controlled with + :ref:`configuration-options.yk8s.openstack.gateway_count`. .. [1] Caveat: Changing the role of a Terraform node - will completely rebuild the node. + will completely rebuild the node. .. attention:: @@ -30,42 +30,42 @@ Consider the following example: openstack = { - - gateway_count = 3; - + gateway_count = 2; # <-- one gateway gets deleted + - gateway_count = 3; + + gateway_count = 2; # <-- one gateway gets deleted - nodes = { + nodes = { worker-0 = { - role = "worker"; - flavor = "M"; - image = "Debian 12 (bookworm)"; + role = "worker"; + flavor = "M"; + image = "Debian 12 (bookworm)"; }; - - worker-1 = { # <-- gets deleted - - role = "worker"; - - flavor = "M"; - - }; + - worker-1 = { # <-- gets deleted + - role = "worker"; + - flavor = "M"; + - }; worker-2 = { - role = "worker"; - flavor = "L"; + role = "worker"; + flavor = "L"; }; - + mon1 = { # <-- gets created - + role = "worker"; - + flavor = "S"; - + image = "Ubuntu 22.04 LTS x64"; - + }; - }; - }; + + mon1 = { # <-- gets created + + role = "worker"; + + flavor = "S"; + + image = "Ubuntu 22.04 LTS x64"; + + }; + }; + }; The name of an OpenStack node is composed from the following parts: - for master/worker nodes: - ``yk8s.infra.cluster_name`` ```` +``yk8s.infra.cluster_name`` ```` - for gateway nodes: - ``yk8s.infra.cluster_name`` ``yk8s.openstack.gateway_defaults.common_name`` ```` +``yk8s.infra.cluster_name`` ``yk8s.openstack.gateway_defaults.common_name`` ```` .. code:: nix - openstack = { + openstack = { cluster_name = "yk8s"; gateway_count = 1; @@ -99,7 +99,7 @@ Availability zones of the underlying Openstack cloud to use for the creation of **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.check_credentials: @@ -131,7 +131,7 @@ to true. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.cinder_enable_topology: @@ -162,7 +162,7 @@ Important: Cinder must support AZs and the AZs must match the AZs used by nova! **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.cinder_volume_type: @@ -186,7 +186,7 @@ of the IaaS-layer is used. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.create_root_disk_on_volume: @@ -215,7 +215,7 @@ Equivalent to ``openstack server create --boot-from-volume […]``. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.dns_nameservers_v4: @@ -236,7 +236,7 @@ A list of IPv4 addresses which will be configured as DNS nameservers of the IPv4 **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.enabled: @@ -258,7 +258,7 @@ Whether to build the cluster on top of Openstack. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.gateway_count: @@ -286,7 +286,7 @@ in which case it will match the amount of availability zones by default. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.gateway_defaults.common_name: @@ -307,7 +307,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.gateway_defaults.flavor: @@ -323,7 +323,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.gateway_defaults.image: @@ -339,7 +339,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.gateway_defaults.root_disk_size: @@ -361,7 +361,7 @@ Only applies if :ref:`configuration-options.yk8s.openstack.create_root_disk_on_v **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.gateway_defaults.root_disk_volume_type: @@ -384,7 +384,7 @@ If null, the default of the IaaS environment will be used. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.keypair: @@ -404,11 +404,11 @@ Will most of the time be set via the environment variable TF_VAR_keypair **Default:**:: - null + "\${var.keypair}" **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.master_defaults.flavor: @@ -424,7 +424,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.master_defaults.image: @@ -440,7 +440,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.master_defaults.root_disk_size: @@ -462,7 +462,7 @@ Only applies if :ref:`configuration-options.yk8s.openstack.create_root_disk_on_v **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.master_defaults.root_disk_volume_type: @@ -485,7 +485,7 @@ If null, the default of the IaaS environment will be used. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.network_mtu: @@ -506,7 +506,7 @@ MTU for the network used for the cluster. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.network_name: @@ -538,7 +538,7 @@ because there might be situations where the CCM should not pick the managed netw **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.nodes: @@ -553,6 +553,8 @@ At least one node with role=master must be given. You may also specify those attributes or a subset of them using :ref:`yk8s.openstack.{master,worker}_defaults `. +Gateways are created automatically, and should not be explicitly added here. + **Type:**:: @@ -565,7 +567,7 @@ using :ref:`yk8s.openstack.{master,worker}_defaults .anti_affinity_group: @@ -588,7 +590,7 @@ If left empty no anti affinity group will be joined. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.nodes..az: @@ -609,7 +611,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.nodes..flavor: @@ -630,7 +632,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.nodes..image: @@ -651,7 +653,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.nodes..role: @@ -663,11 +665,11 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Type:**:: - one of "master", "worker" + one of "master", "worker", "gateway" **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.nodes..root_disk_size: @@ -688,7 +690,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.nodes..root_disk_volume_type: @@ -709,7 +711,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.public_network: @@ -726,7 +728,7 @@ Name of the Openstack provider network to use **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.spread_gateways_across_azs: @@ -747,7 +749,7 @@ If true, spawn a gateway node in each availability zone listed in :ref:`configur **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.thanos_delete_container: @@ -778,7 +780,7 @@ are switched off **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.worker_defaults.anti_affinity_group: @@ -800,7 +802,7 @@ Leaving this empty means to not join any anti affinity group **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.worker_defaults.flavor: @@ -816,7 +818,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.worker_defaults.image: @@ -832,7 +834,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.ni **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.worker_defaults.root_disk_size: @@ -854,7 +856,7 @@ Only applies if :ref:`configuration-options.yk8s.openstack.create_root_disk_on_v **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack .. _configuration-options.yk8s.openstack.worker_defaults.root_disk_volume_type: @@ -877,5 +879,5 @@ If null, the default of the IaaS environment will be used. **Declared by** -https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack.nix +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/openstack diff --git a/docs/user/reference/options/yk8s.terraform.rst b/docs/user/reference/options/yk8s.terraform.rst index dc584a2cbff750267d191dcd3fa6706ba874cbdc..30ecc6fc17af14a60ad30aa5b4f08b440104ff7d 100644 --- a/docs/user/reference/options/yk8s.terraform.rst +++ b/docs/user/reference/options/yk8s.terraform.rst @@ -89,7 +89,7 @@ https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/terraform.ni Whether to enable GitLab-managed Terraform backend If true, the Terraform state will be stored inside the provided gitlab project. -If set, the environment `TF_HTTP_USERNAME` and `TF_HTTP_PASSWO = mkOptionD` +If set, the environment `TF_HTTP_USERNAME` and `TF_HTTP_PASSWORD` must be configured in a separate file `~/.config/yaook-k8s/env`. . @@ -188,6 +188,27 @@ The name of the Gitlab state object in which to store the Terraform state, e.g. https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/terraform.nix +.. _configuration-options.yk8s.terraform.modules: + +``yk8s.terraform.modules`` +########################## + + + +**Type:**:: + + list of anything + + +**Default:**:: + + [ ] + + +**Declared by** +https://gitlab.com/alasca.cloud/tarook/tarook/-/tree/devel/nix/yk8s/terraform.nix + + .. _configuration-options.yk8s.terraform.timeout_time: ``yk8s.terraform.timeout_time`` diff --git a/flake.lock b/flake.lock index bb6e8973399646835e1b9be013e064b13f88f9a9..38097f3f2b7a9cd4d052232aa965d70588097f26 100644 --- a/flake.lock +++ b/flake.lock @@ -18,6 +18,27 @@ "type": "github" } }, + "flake-parts_2": { + "inputs": { + "nixpkgs-lib": [ + "terranix", + "nixpkgs" + ] + }, + "locked": { + "lastModified": 1736143030, + "narHash": "sha256-+hu54pAoLDEZT9pjHlqL9DNzWz0NbUn8NEAHP7PQPzU=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "b905f6fc23a9051a6e1b741e1438dbfc0634c6de", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, "nixpkgs": { "locked": { "lastModified": 1757020766, @@ -97,6 +118,21 @@ "type": "github" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1728956102, + "narHash": "sha256-J8zo+UYNjHATsxn2/ROl8iaji2RgLm+sG7b3VcD36YM=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "3d85bae2431f20ab1ac5cf14d03d314dffe629af", + "type": "github" + }, + "original": { + "owner": "nixos", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { "flake-parts": "flake-parts", @@ -104,7 +140,8 @@ "nixpkgs-terraform157": "nixpkgs-terraform157", "nixpkgs-unstable": "nixpkgs-unstable", "nixpkgs-vault1148": "nixpkgs-vault1148", - "systems": "systems" + "systems": "systems", + "terranix": "terranix" } }, "systems": { @@ -122,6 +159,41 @@ "rev": "2ecfcac5e15790ba6ce360ceccddb15ad16d08a8", "type": "github" } + }, + "systems_2": { + "locked": { + "lastModified": 1681028828, + "narHash": "sha256-Vy1rq5AaRuLzOxct8nz4T6wlgyUR7zLU309k9mBC768=", + "owner": "nix-systems", + "repo": "default", + "rev": "da67096a3b9bf56a91d16901293e51ba5b49a27e", + "type": "github" + }, + "original": { + "owner": "nix-systems", + "repo": "default", + "type": "github" + } + }, + "terranix": { + "inputs": { + "flake-parts": "flake-parts_2", + "nixpkgs": "nixpkgs_2", + "systems": "systems_2" + }, + "locked": { + "lastModified": 1757278723, + "narHash": "sha256-hTMi6oGU+6VRnW9SZZ+muFcbfMEf2ajjOp7Z2KM5MMY=", + "owner": "terranix", + "repo": "terranix", + "rev": "924573fa6587ac57b0d15037fbd2d3f0fcdf17fb", + "type": "github" + }, + "original": { + "owner": "terranix", + "repo": "terranix", + "type": "github" + } } }, "root": "root", diff --git a/flake.nix b/flake.nix index e1a1aa0fbf7def525b86e5a42f5d26f858969e30..538223bc0c24feeaa1c92597b23477dc51510648 100644 --- a/flake.nix +++ b/flake.nix @@ -5,6 +5,7 @@ inputs.nixpkgs-unstable.url = "github:NixOS/nixpkgs/nixos-unstable"; inputs.flake-parts.url = "github:hercules-ci/flake-parts"; inputs.systems.url = "github:nix-systems/x86_64-linux/2ecfcac5e15790ba6ce360ceccddb15ad16d08a8"; + inputs.terranix.url = "github:terranix/terranix"; outputs = inputs @ { self, @@ -51,7 +52,7 @@ formatter = pkgs.alejandra; }; flake = {lib, ...}: { - flakeModules.yk8s = import ./nix/yk8s; + flakeModules.yk8s = flake-parts.lib.importApply ./nix/yk8s {localFlake = self;}; }; }; } diff --git a/nix/templates/module/example.nix b/nix/templates/module/example.nix index 43843da12a856049a9c54c283385e9ea39638312..109c7f53755c8134148147e77efcb5dcd07c5698 100644 --- a/nix/templates/module/example.nix +++ b/nix/templates/module/example.nix @@ -41,7 +41,7 @@ in { example = ["some value" "some other value"]; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; inventory_path = "all/example.yaml"; # this is where the values will end up under inventory/group_vars diff --git a/nix/yk8s/conf-vars.nix b/nix/yk8s/conf-vars.nix new file mode 100644 index 0000000000000000000000000000000000000000..4ec5d2946daad9cf1a62aa539d9b93764075d3bc --- /dev/null +++ b/nix/yk8s/conf-vars.nix @@ -0,0 +1,21 @@ +{ + config, + lib, + yk8s-lib, + ... +}: let + inherit (lib) mkOption types; + inherit (yk8s-lib) mkTopSection mkGroupVarsFile; +in { + config.yk8s._targets.conf_vars = { + inventory_subdir = "conf_vars"; + inventory_packages = [ + (yk8s-lib.mkYamlAtPath "main.yaml" { + tf_usage = config.yk8s.terraform.enabled; + wg_usage = config.yk8s.wireguard.enabled; + wg_subnet = config.yk8s.infra.subnet_cidr; + wg_subnet_v6 = config.yk8s.infra.subnet_v6_cidr; + }) + ]; + }; +} diff --git a/nix/yk8s/containerd.nix b/nix/yk8s/containerd.nix index d620f51938ae3faddafebf667f4eac53d805066c..e0cee5d7dfcb6bc491e15aa93f47c91d9b991a52 100644 --- a/nix/yk8s/containerd.nix +++ b/nix/yk8s/containerd.nix @@ -60,7 +60,7 @@ in { ]; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "containerd_"; diff --git a/nix/yk8s/custom.nix b/nix/yk8s/custom.nix index 4fa687dca5b473fa9df328b556cbcd6358efebf6..0cf54197329693920bd8ac547caea2aafaf855a7 100644 --- a/nix/yk8s/custom.nix +++ b/nix/yk8s/custom.nix @@ -23,7 +23,7 @@ in { }; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; inventory_path = "all/custom.yaml"; diff --git a/nix/yk8s/default.nix b/nix/yk8s/default.nix index 5d2bccba9dd2cd795d5051f7359bafe864f3b3ef..1c9cf38d34d4156661c1243f0aaf7f663a81100c 100644 --- a/nix/yk8s/default.nix +++ b/nix/yk8s/default.nix @@ -1,4 +1,4 @@ -{ +{localFlake}: { inputs, lib, self, @@ -29,16 +29,18 @@ in { config._module.args = { inherit yk8s-lib; + terranix-lib = localFlake.inputs.terranix.lib; # Pin all packages used by this module to the version managed in the Tarook repo - pkgs = import inputs.yk8s.inputs.nixpkgs { + pkgs = import localFlake.inputs.nixpkgs { inherit system; }; }; imports = [ ./assertions.nix + ./conf-vars.nix ./infra.nix ./terraform.nix - ./openstack.nix + ./openstack ./vault.nix ./load-balancing.nix ./kubernetes @@ -70,7 +72,7 @@ Base path to the Ansible inventory. Files will get written here. ''; type = relativePosixPath; - default = "inventory/yaook-k8s"; + default = "inventory"; }; _state_base_path = mkOption { description = '' @@ -79,38 +81,76 @@ type = relativePosixPath; default = "state"; }; - _inventory_packages = mkInternalOption { - description = '' - Inventory packages from all sections that are then merged into the inventory directory - ''; - type = with types; listOf package; - default = []; - }; - _state_packages = mkInternalOption { - description = '' - State packages from all sections that are then merged into the state directory - ''; - type = with types; listOf package; - default = []; + _targets = mkInternalOption { + type = with types; + attrsOf (submodule { + options = { + inventory_subdir = mkInternalOption { + description = '' + The directory inside _inventory_base_path in which inventory packages are to be created. + ''; + type = with types; nullOr nonEmptyStr; + }; + inventory_packages = mkInternalOption { + description = '' + Inventory packages from all sections that are then merged into the inventory directory + ''; + type = with types; listOf package; + default = []; + }; + state_packages = mkInternalOption { + description = '' + State packages from all sections that are then merged into the state directory + ''; + type = with types; listOf package; + default = []; + }; + }; + }); }; }; - config.packages = rec { - yk8s-inventory = pkgs.buildEnv { - name = "yaook-k8s-inventory"; - paths = cfg._inventory_packages; - }; - yk8s-state-dir = pkgs.buildEnv { - name = "yaook-k8s-state-dir"; - paths = cfg._state_packages; + config.yk8s._targets.ansible.inventory_subdir = "yaook-k8s"; + config.yk8s.assertions = + lib.mapAttrsToList (targetName: targetOptions: { + assertion = (targetOptions.inventory_packages != []) -> targetOptions.inventory_subdir != null; + message = "Target ${targetName} has inventory_packages, but inventory_subdir is not defined."; + }) + cfg._targets; + config.packages = lib.foldlAttrs (acc: targetName: targetOptions: let + hasInventory = targetOptions.inventory_packages != []; + inventory = pkgs.buildEnv { + name = "yk8s-outputs-${targetName}-inventory"; + paths = targetOptions.inventory_packages; }; - yk8s-outputs = builtins.seq (baseSystemAssertWarn config.yk8s) pkgs.buildEnv { - name = "yaook-k8s-outputs"; - paths = [ - (linkToPath yk8s-inventory cfg._inventory_base_path) - (linkToPath yk8s-state-dir cfg._state_base_path) - ]; + state-dir = pkgs.buildEnv { + name = "yk8s-outputs-${targetName}-state-dir"; + paths = targetOptions.state_packages; }; - }; + in + acc + // { + "yk8s-outputs-${targetName}" = builtins.seq (baseSystemAssertWarn config.yk8s) pkgs.buildEnv { + name = "yk8s-outputs-${targetName}"; + paths = let + inventoryPath = "${cfg._inventory_base_path}/${targetOptions.inventory_subdir}"; + in + [ + (pkgs.writeTextDir ".path-info" + '' + inventory=${ + if hasInventory + then inventoryPath + else "" + } + state=${cfg._state_base_path} + '') + (linkToPath state-dir cfg._state_base_path) + ] + ++ lib.optional hasInventory + (linkToPath inventory inventoryPath); + }; + }) {} + cfg._targets; }); }; } diff --git a/nix/yk8s/infra.nix b/nix/yk8s/infra.nix index bc3d81f2f6ab6484d3037caa589b3b70e34eef9c..c1bece8f9965956e71cdb4632ebe36245de061ef 100644 --- a/nix/yk8s/infra.nix +++ b/nix/yk8s/infra.nix @@ -10,7 +10,7 @@ inherit (modules-lib) mkRemovedOptionModule; inherit (pkgs.stdenv) mkDerivation; inherit (lib) mkEnableOption mkOption types; - inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkDisableOption linkToPath; + inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkDisableOption linkToPath mkYamlAtPath mkInternalOption; inherit (yk8s-lib.types) ipv4Addr @@ -18,6 +18,7 @@ ipv4Cidr ipv6Cidr k8sClusterName + absolutePosixPath ; in { options.yk8s.infra = mkTopSection { @@ -71,10 +72,10 @@ in { type = types.nullOr ipv4Addr; default = null; apply = v: - if v == null && cfg.ipv4_enabled && config.yk8s.openstack.enabled == false - then throw "config.yk8s.infra.networking_fixed_ip must be set if config.yk8s.infra.ipv4_enabled=true and config.yk8s.openstack.enabled=false" - else if v != null && config.yk8s.openstack.enabled == true - then throw "config.yk8s.infra.networking_fixed_ip must not be set if config.yk8s.openstack.enabled=true" + if cfg.ipv4_enabled && v == null + then + throw + "config.yk8s.infra.networking_fixed_ip must be set if ipv4 is enabled" else v; }; @@ -82,38 +83,176 @@ in { type = with types; nullOr ipv6Addr; default = null; apply = v: - if v == null && cfg.ipv6_enabled && config.yk8s.openstack.enabled == false - then throw "config.yk8s.infra.networking_fixed_ip_v6 must be set if config.yk8s.infra.ipv6_enabled=true and config.yk8s.openstack.enabled=false" - else if v != null && config.yk8s.openstack.enabled == true - then throw "config.yk8s.infra.networking_fixed_ip_v6 must not be set if config.yk8s.openstack.enabled=true" + if cfg.ipv6_enabled && v == null + then + throw + "config.yk8s.infra.networking_fixed_ip_v6 must be set if ipv6 is enabled" else v; }; + networking_floating_ip = mkInternalOption { + # TODO: move to yk8s.wireguard when ipsec gets removed + description = '' + Address that is used by Wireguard and IPsec to connect to the active gateway node. + ''; + type = types.nullOr ipv4Addr; + default = null; + }; + hosts_file = mkOption { description = '' - A custom hosts file in case :ref:`configuration-options.yk8s.openstack.enabled` is set to ``false`` + A custom hosts file. This option is deprecated. Use :ref:`configuration-options.yk8s.infra.ansible_hosts` instead. ''; type = with types; nullOr pathInStore; default = null; example = lib.options.literalExpression "./hosts"; - apply = v: - if v == null && config.yk8s.openstack.enabled == false - then throw "infra.hosts_file must be set if openstack is disabled" - else if v != null && config.yk8s.openstack.enabled == true - then throw "infra.hosts_file must not be set if openstack is enabled" - else v; + }; + + ansible_hosts = let + applyGroupSubmoduleAttrs = lib.mapAttrs (_: lib.filterAttrs (_: a: a != {})); + groupSubmodule = types.submodule { + options = { + children = mkOption { + visible = "shallow"; # Otherwise renderDocs chokes on the recursive submodule + type = types.attrsOf groupSubmodule; + default = {}; + apply = applyGroupSubmoduleAttrs; + }; + hosts = mkOption { + type = with types; attrsOf anything; + default = {}; + }; + vars = mkOption { + type = with types; attrsOf anything; + default = {}; + }; + }; + }; + in + mkOption { + description = '' + Entries to the Ansible hosts file. Will be rendered to a YAML-based file into the inventory. + This option is mandatory for bare-metal clusters and is automatically managed if Terraform is used. + + Check the parts regarding YAML in the Ansible documentation: https://docs.ansible.com/ansible/latest/inventory_guide/intro_inventory.html + ''; + default = null; + apply = applyGroupSubmoduleAttrs; + type = types.nullOr (types.submodule { + freeformType = types.attrsOf groupSubmodule; + options = { + all.vars.ansible_python_interpreter = mkOption { + type = absolutePosixPath; + default = "/usr/bin/python3"; + }; + frontend = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + default = {children.gateways = {};}; + example = {children.masters = {};}; + }; + gateways = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + default = {}; + }; + k8s_nodes = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + default = { + children = { + masters = {}; + workers = {}; + }; + }; + }; + masters = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + example = { + hosts = { + devcluster-master-1 = { + ansible_host = "172.30.154.66"; + local_ipv4_address = "172.30.154.66"; + }; + }; + }; + }; + workers = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + default = {}; + example = { + hosts = { + devcluster-worker-1 = { + ansible_host = "172.30.154.99"; + local_ipv4_address = "172.30.154.99"; + }; + }; + }; + }; + orchestrator = mkOption { + visible = "shallow"; # Otherwise the submodule's options are repeated here + type = groupSubmodule; + default = { + hosts.localhost = { + ansible_connection = "local"; + ansible_python_interpreter = "{{ ansible_playbook_python }}"; + }; + }; + }; + }; + }); + }; + + final_hosts = mkInternalOption { + description = '' + Internal read-only option to access all hosts and their effective attributes available to Ansible. + Each host gets the additional attribute ``group_names`` which is a list of all Ansible groups to which the host belongs. + ''; + readOnly = true; + type = with types; nullOr attrs; + default = + if cfg.ansible_hosts == null + then null + else let + getHostsFromGroupAttrs = lib.foldlAttrs ( + acc: groupName: groupValues: let + hosts = lib.recursiveUpdate (getHostsFromGroupAttrs (groupValues.children or {})) (groupValues.hosts or {}); + in + lib.recursiveUpdate acc (lib.mapAttrs ( + hostName: hostValues: hostValues // {group_names = (hostValues.group_names or []) ++ [groupName];} + ) + hosts) + ) {}; + in + getHostsFromGroupAttrs cfg.ansible_hosts; }; }; - config.yk8s._inventory_packages = - [ + + config.yk8s.assertions = [ + { + assertion = (config.yk8s.wireguard.enabled || config.yk8s.ipsec.enabled) -> config.yk8s.terraform.enabled || cfg.networking_floating_ip != null; + message = "config.yk8s.infra.networking_floating_ip must be set if Wireguard or IPsec is used."; + } + { + assertion = cfg.ansible_hosts != null -> cfg.hosts_file == null; + message = "config.yk8s.infra.hosts_file must not be set if config.yk8s.infra.ansible_hosts is used (which implicitly happens through Terraform)."; + } + { + assertion = ! config.yk8s.terraform.enabled -> (cfg.ansible_hosts == null && cfg.hosts_file == null); + message = "One of config.yk8s.infra.hosts_file and config.yk8s.infra.ansible_hosts must be set"; + } + ]; + config.yk8s.warnings = lib.optional (cfg.hosts_file != null) "config.yk8s.infra.hosts_file is deprecated. Use config.yk8s.infra.ansible_hosts instead."; + config.yk8s._targets.ansible.inventory_packages = + (lib.optional (cfg.ansible_hosts != null) (mkYamlAtPath "hosts" cfg.ansible_hosts)) + ++ (lib.optional (cfg.hosts_file != null) (linkToPath cfg.hosts_file "hosts")) + ++ [ (mkGroupVarsFile { inherit cfg; inventory_path = "all/infra.yaml"; - transformations = - [(lib.attrsets.filterAttrs (n: _: n != "hosts_file"))] - ++ (lib.optional config.yk8s.openstack.enabled (lib.attrsets.filterAttrs (n: _: ! (builtins.elem n ["networking_fixed_ip" "networking_fixed_ip_v6"])))); + transformations = [(c: removeAttrs c ["hosts_file" "ansible_hosts"])]; }) - ] - ++ lib.optional (cfg.hosts_file != null) - (linkToPath cfg.hosts_file "hosts"); + ]; } diff --git a/nix/yk8s/k8s-supplements/cert-manager.nix b/nix/yk8s/k8s-supplements/cert-manager.nix index bfca15e500ebf01c26d653921737d710556edb82..0c9af48b6e41b9c220b5c66b4af20d715c070b98 100644 --- a/nix/yk8s/k8s-supplements/cert-manager.nix +++ b/nix/yk8s/k8s-supplements/cert-manager.nix @@ -112,7 +112,7 @@ in { example = "https://acme-staging-v02.api.letsencrypt.org/directory"; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ ( mkGroupVarsFile { inherit cfg; diff --git a/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix b/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix index e244de4d6d9817c9e7c7694d767f44a3089bac71..4cf1cac93963ec08106d0ba2d2f1556f9a5d9232 100644 --- a/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix +++ b/nix/yk8s/k8s-supplements/ch-k8s-lbaas.nix @@ -8,7 +8,7 @@ modules-lib = import ../lib/modules.nix {inherit lib;}; inherit (modules-lib) mkRenamedResourceOptionModule mkResourceOptionModule; inherit (lib) mkOption mkEnableOption types; - inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkResourceOption mkDisableOption; + inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkInternalOption mkDisableOption; inherit (yk8s-lib.types) base64Str @@ -124,6 +124,26 @@ in { type = types.bool; default = config.yk8s.kubernetes.network.calico.enabled; }; + + subnet_id = mkInternalOption { + type = with types; nullOr nonEmptyStr; + default = null; + + apply = v: + if config.yk8s.openstack.enabled && v == null + then throw "ch-k8s-lbaas.subnet_id must be set if openstack is enabled" + else v; + }; + floating_ip_network_id = mkInternalOption { + type = with types; nullOr nonEmptyStr; + default = null; + apply = v: + if config.yk8s.openstack.enabled && v == null + then + throw + "ch-k8s-lbaas.floating_ip_network_id must be set if openstack is enabled" + else v; + }; }; config.yk8s.assertions = [ # Due to OVN support, require version >= 0.8.0 (warn only if not in semver2 format) @@ -169,8 +189,12 @@ in { message = "config.yk8s.ch-k8s-lbaas.version: must be at least 0.8.0"; } ) + { + assertion = (!config.yk8s.openstack.enabled) -> (cfg.subnet_id == null && cfg.floating_ip_network_id == null); + message = "config.yk8s.ch-k8s-lbaas.subnet_id and config.yk8s.ch-k8s-lbaas.floating_ip_network_id must be null if config.yk8s.openstack.enabled==false"; + } ]; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "ch_k8s_lbaas_"; diff --git a/nix/yk8s/k8s-supplements/etcd-backup.nix b/nix/yk8s/k8s-supplements/etcd-backup.nix index f7810c19f0dfd5fc184accd748f5f43d1602547f..b5dfb25ce26a9ccb32ae963a9b69bd0710e008d1 100644 --- a/nix/yk8s/k8s-supplements/etcd-backup.nix +++ b/nix/yk8s/k8s-supplements/etcd-backup.nix @@ -243,7 +243,7 @@ in { }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; unflat = [ diff --git a/nix/yk8s/k8s-supplements/fluxcd.nix b/nix/yk8s/k8s-supplements/fluxcd.nix index e9252b3075721fe26aa3d6237c7f7f6daeeade45..4bd4257550bf443e6fc538ada17a15aa44abea47 100644 --- a/nix/yk8s/k8s-supplements/fluxcd.nix +++ b/nix/yk8s/k8s-supplements/fluxcd.nix @@ -66,7 +66,7 @@ in { default = null; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "fluxcd_"; diff --git a/nix/yk8s/k8s-supplements/ingress.nix b/nix/yk8s/k8s-supplements/ingress.nix index e2c74a5994b7edd016c2a464d764eccb4e965131..ee50bd2e7cead967895c8d5c9c56801af070a5c9 100644 --- a/nix/yk8s/k8s-supplements/ingress.nix +++ b/nix/yk8s/k8s-supplements/ingress.nix @@ -169,7 +169,7 @@ in { }; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; unflat = [["helm" "values"]]; diff --git a/nix/yk8s/k8s-supplements/ipsec.nix b/nix/yk8s/k8s-supplements/ipsec.nix index 35be06baed301b74b0e53bf320738ee51ab9e99c..df8ecceb0b03eb2db4281d9555a0f89feac47a78 100644 --- a/nix/yk8s/k8s-supplements/ipsec.nix +++ b/nix/yk8s/k8s-supplements/ipsec.nix @@ -160,7 +160,7 @@ in { } ]; config.yk8s.warnings = lib.optional (cfg.enabled) "config.yk8s.ipsec: is deprecated. Support for it will be dropped in a release after v11.0.0"; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "ipsec_"; diff --git a/nix/yk8s/k8s-supplements/monitoring.nix b/nix/yk8s/k8s-supplements/monitoring.nix index 2b487a015dd829aecd4d912e703332105be054cd..3745b5cacdd8e7c5ff583800adfaacba12a54731 100644 --- a/nix/yk8s/k8s-supplements/monitoring.nix +++ b/nix/yk8s/k8s-supplements/monitoring.nix @@ -582,7 +582,7 @@ in { message = "config.yk8s.k8s-service-layer.prometheus.internet_probe_targets[${idx}].module: ${x.module} is an IPv6-specific module but config.yk8s.infra.ipv6_enabled=false"; }) cfg.internet_probe_targets; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "monitoring_"; diff --git a/nix/yk8s/k8s-supplements/rook.nix b/nix/yk8s/k8s-supplements/rook.nix index 156665711420302edc4dbafeeb4fb47baababdd0..6973b06c14e8d0c981cd575658e655a62738dc92 100644 --- a/nix/yk8s/k8s-supplements/rook.nix +++ b/nix/yk8s/k8s-supplements/rook.nix @@ -422,7 +422,7 @@ in { }); }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "rook_"; diff --git a/nix/yk8s/k8s-supplements/vault.nix b/nix/yk8s/k8s-supplements/vault.nix index a49cd0b5b1a899d83668b5b6483a49e3e3a274a1..fda5ea2c510e7ba7ae837f8610ebf0aaff18f958 100644 --- a/nix/yk8s/k8s-supplements/vault.nix +++ b/nix/yk8s/k8s-supplements/vault.nix @@ -250,7 +250,7 @@ in { warnIfZero "config.yk8s.k8s-service-layer.vault.service_active_node_port: should not be port zero" v; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "yaook_vault_"; diff --git a/nix/yk8s/k8s-supplements/wireguard/default.nix b/nix/yk8s/k8s-supplements/wireguard/default.nix index 9be70d16a871d4da9746902e10de230092624077..3bb606d6f97519908f7c44b9d2887eaa5b7e2267 100644 --- a/nix/yk8s/k8s-supplements/wireguard/default.nix +++ b/nix/yk8s/k8s-supplements/wireguard/default.nix @@ -246,7 +246,7 @@ in { ${wireguard_helper}/bin/wireguard_helper ${varsFile} $out/${inventory_path} ''; in { - _inventory_packages = + _targets.ansible.inventory_packages = if cfg.enabled then [(linkToPath "${wireguard_helper_output}/${inventory_path}" "group_vars/${inventory_path}")] else [ @@ -255,7 +255,7 @@ in { inherit inventory_path; }) ]; - _state_packages = lib.lists.optional cfg.enabled (linkToPath "${wireguard_helper_output}/${ipam_path}" ipam_path); + _targets.ansible.state_packages = lib.lists.optional cfg.enabled (linkToPath "${wireguard_helper_output}/${ipam_path}" ipam_path); warnings = lib.optional (cfg.enabled && (builtins.length cfg.peers) == 0) "config.yk8s.wireguard.peers: is empty"; assertions = let inherit (builtins) length; diff --git a/nix/yk8s/kubernetes/default.nix b/nix/yk8s/kubernetes/default.nix index 5541617378e0b3878ced5567f86bf4b9b4f6c69b..b439513acbe3bb7e47ca3a324cda93ac972c4776 100644 --- a/nix/yk8s/kubernetes/default.nix +++ b/nix/yk8s/kubernetes/default.nix @@ -173,7 +173,7 @@ in { message = "config.yk8s.kubernetes.is_gpu_cluster: is mutually exlusive with config.yk8s.kubernetes.virtualize_gpu"; } ]; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "k8s_"; diff --git a/nix/yk8s/lib/default.nix b/nix/yk8s/lib/default.nix index 8bb822c5bd7bdc9ddfbde3a3cad8cccb93eb0234..e3cd83b33684252b6b3f9bbb2db4797fe7dd80f2 100644 --- a/nix/yk8s/lib/default.nix +++ b/nix/yk8s/lib/default.nix @@ -31,6 +31,8 @@ cat ${preamble} ${file} > $out ''; + tfRef = ref: "\${${ref}}"; + mkGroupVarsFile = { cfg, inventory_path, diff --git a/nix/yk8s/load-balancing.nix b/nix/yk8s/load-balancing.nix index 3057ee23eb7d11389486e745afa7a8097760d817..78b324070c0e4e26923d90f1b438ef65b891d6d5 100644 --- a/nix/yk8s/load-balancing.nix +++ b/nix/yk8s/load-balancing.nix @@ -140,7 +140,7 @@ in { default = 2000; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; inventory_path = "all/load-balancing.yaml"; diff --git a/nix/yk8s/miscellaneous.nix b/nix/yk8s/miscellaneous.nix index 9776f441031e854d37a8d3ef7166e7e3baeed820..e0b900ef005dceaa468d6edc3abe5f84bf2d7682 100644 --- a/nix/yk8s/miscellaneous.nix +++ b/nix/yk8s/miscellaneous.nix @@ -178,7 +178,7 @@ in { message = "config.yk8s.miscellaneous.no_proxy: must be set because config.yk8s.miscellaneous.cluster_behind_proxy=true"; } ]; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; inventory_path = "all/miscellaneous.yaml"; diff --git a/nix/yk8s/node-scheduling.nix b/nix/yk8s/node-scheduling.nix index 3a44f767ed06652ffc4b8b2e6c14360f4e453dce..3da1ebefb85fab91d2fdb8c50203013bf14afb44 100644 --- a/nix/yk8s/node-scheduling.nix +++ b/nix/yk8s/node-scheduling.nix @@ -8,7 +8,6 @@ cfg = config.yk8s.node-scheduling; inherit (lib) mkOption types; inherit (yk8s-lib) mkTopSection mkGroupVarsFile; - nodeNames = map (n: "${config.yk8s.infra.cluster_name}-${n}") (builtins.attrNames config.yk8s.openstack.nodes); inherit (yk8s-lib.types) k8sLabelStr @@ -71,12 +70,6 @@ in { "''${scheduling_key_prefix}/monitoring=true" ]; }''; - apply = v: - builtins.seq (builtins.all (e: - if config.yk8s.terraform.enabled -> builtins.elem e nodeNames - then true - else throw "config.yk8s.node-scheduling.labels: label defined for ${e}, but node not found in config.yk8s.openstack.nodes") (builtins.attrNames v)) - v; }; taints = mkOption { description = '' @@ -96,15 +89,18 @@ in { "''${scheduling_key_prefix}/storage=true:NoSchedule" ]; }''; - apply = v: - builtins.seq (builtins.all (e: - if config.yk8s.terraform.enabled -> builtins.elem e nodeNames - then true - else throw "config.yk8s.node-scheduling.taints: taint defined for ${e}, but node not found in config.yk8s.openstack.nodes") (builtins.attrNames v)) - v; }; }; - config.yk8s._inventory_packages = [ + config.yk8s.warnings = + (builtins.foldl' (acc: e: + acc + ++ lib.optional (config.yk8s.infra.final_hosts != null && ! builtins.hasAttr e config.yk8s.infra.final_hosts) + "config.yk8s.node-scheduling.labels: label defined for ${e}, but node not found in config.yk8s.infra.ansible_hosts") [] (builtins.attrNames cfg.labels)) + ++ (builtins.foldl' (acc: e: + acc + ++ lib.optional (config.yk8s.infra.final_hosts != null && ! builtins.hasAttr e config.yk8s.infra.final_hosts) + "config.yk8s.node-scheduling.taints: taint defined for ${e}, but node not found in config.yk8s.infra.ansible_hosts") [] (builtins.attrNames cfg.taints)); + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; inventory_path = "all/node-scheduling.yaml"; diff --git a/nix/yk8s/nvidia.nix b/nix/yk8s/nvidia.nix index 7cfe7615470bc383dfa54c67906b5b8efd068470..440a09aa39e7c0d136378c2d4ac9adca98069f80 100644 --- a/nix/yk8s/nvidia.nix +++ b/nix/yk8s/nvidia.nix @@ -30,7 +30,7 @@ in { }; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "nvidia_"; diff --git a/nix/yk8s/openstack/00-backend.nix b/nix/yk8s/openstack/00-backend.nix new file mode 100644 index 0000000000000000000000000000000000000000..630faf8ffede84321bdfa6cdc51b2a5ba8b744ce --- /dev/null +++ b/nix/yk8s/openstack/00-backend.nix @@ -0,0 +1,25 @@ +{ + lib, + config, + ... +}: { + yk8s.terraform.modules = lib.singleton { + terraform.backend = + if config.yk8s.terraform.gitlab_backend + then { + "http" = rec { + address = config.yk8s.terraform.gitlab_backend_address; + lock_address = "${address}/lock"; + unlock_address = "${address}/lock"; + lock_method = "POST"; + unlock_method = "DELETE"; + retry_wait_min = 5; + }; + } + else { + "local" = { + path = "terraform.tfstate"; + }; + }; + }; +} diff --git a/nix/yk8s/openstack/00-provider.nix b/nix/yk8s/openstack/00-provider.nix new file mode 100644 index 0000000000000000000000000000000000000000..6c82f48dea91cde0bfb561fd4df6bf12c124ad9a --- /dev/null +++ b/nix/yk8s/openstack/00-provider.nix @@ -0,0 +1,5 @@ +{lib, ...}: { + yk8s.terraform.modules = lib.singleton { + provider.openstack = {}; + }; +} diff --git a/nix/yk8s/openstack/00-variables.nix b/nix/yk8s/openstack/00-variables.nix new file mode 100644 index 0000000000000000000000000000000000000000..0cf293bc90dac25def1659ddf2820e8204ff1914 --- /dev/null +++ b/nix/yk8s/openstack/00-variables.nix @@ -0,0 +1,5 @@ +{lib, ...}: { + yk8s.terraform.modules = lib.singleton { + variable.keypair.type = "string"; + }; +} diff --git a/nix/yk8s/openstack/00-versions.nix b/nix/yk8s/openstack/00-versions.nix new file mode 100644 index 0000000000000000000000000000000000000000..e5aa37ba4dc2224b540a741adfbcdd6c7222838a --- /dev/null +++ b/nix/yk8s/openstack/00-versions.nix @@ -0,0 +1,19 @@ +{lib, ...}: { + yk8s.terraform.modules = lib.singleton { + terraform.required_providers = { + local = { + source = "hashicorp/local"; + version = ">= 2.4.0"; + }; + openstack = { + source = "terraform-provider-openstack/openstack"; + version = "~> 3.3.0"; + }; + template = { + source = "hashicorp/template"; + version = ">= 2.2.0"; + }; + }; + # required_version = ">= 0.14"; + }; +} diff --git a/nix/yk8s/openstack/01-discovery.nix b/nix/yk8s/openstack/01-discovery.nix new file mode 100644 index 0000000000000000000000000000000000000000..8fa84e24921b4b116a8bb21b97b333634b05845a --- /dev/null +++ b/nix/yk8s/openstack/01-discovery.nix @@ -0,0 +1,29 @@ +{ + lib, + yk8s-lib, + config, + ... +}: let + cfg = config.yk8s.openstack; +in { + yk8s.terraform.modules = lib.singleton { + data."openstack_networking_network_v2"."public_network" = { + name = cfg.public_network; + }; + + data."openstack_compute_flavor_v2"."gateway" = { + name = cfg.gateway_defaults.flavor; + }; + + data."openstack_images_image_v2"."gateway" = { + name = cfg.gateway_defaults.image; + most_recent = true; + }; + + output.floating_ip_network_id = [ + { + value = yk8s-lib.tfRef "data.openstack_networking_network_v2.public_network.id"; + } + ]; + }; +} diff --git a/nix/yk8s/openstack/10-networking.nix b/nix/yk8s/openstack/10-networking.nix new file mode 100644 index 0000000000000000000000000000000000000000..760b8f87f6031ae34ceba9b5cc6c89eedce9e985 --- /dev/null +++ b/nix/yk8s/openstack/10-networking.nix @@ -0,0 +1,267 @@ +{ + lib, + yk8s-lib, + config, + ... +}: let + cfg = config.yk8s.openstack; +in { + yk8s.terraform.modules = + [ + { + resource.openstack_networking_network_v2.cluster_network = [ + { + admin_state_up = true; + lifecycle = [ + { + ignore_changes = [ + "mtu" + ]; + } + ]; + mtu = cfg.network_mtu; + name = "${config.yk8s.infra.cluster_name}-network"; + } + ]; + resource.openstack_networking_router_v2.cluster_router = [ + { + admin_state_up = true; + external_network_id = yk8s-lib.tfRef "data.openstack_networking_network_v2.public_network.id"; + name = "${config.yk8s.infra.cluster_name}-router"; + } + ]; + + resource.openstack_networking_secgroup_v2.barndoor = [ + { + description = "A barndoor wide open"; + name = "barndoor"; + } + ]; + } + ] + ++ (lib.optional config.yk8s.infra.ipv4_enabled { + resource.openstack_networking_router_interface_v2.cluster_router_iface = { + _import_from = "openstack_networking_router_interface_v2.cluster_router_iface[0]"; + router_id = yk8s-lib.tfRef "openstack_networking_router_v2.cluster_router.id"; + subnet_id = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_subnet.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv4-icmp-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv4-icmp-egress[0]"; + direction = "egress"; + ethertype = "IPv4"; + protocol = "icmp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv4-icmp-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv4-icmp-ingress[0]"; + direction = "ingress"; + ethertype = "IPv4"; + protocol = "icmp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv4-vrrp-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv4-vrrp-ingress[0]"; + direction = "ingress"; + ethertype = "IPv4"; + protocol = "vrrp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv4-vrrp-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv4-vrrp-egress[0]"; + direction = "egress"; + ethertype = "IPv4"; + protocol = "vrrp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv4-tcp-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv4-tcp-egress[0]"; + direction = "egress"; + ethertype = "IPv4"; + port_range_max = 0; + port_range_min = 0; + protocol = "tcp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv4-tcp-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv4-tcp-ingress[0]"; + direction = "ingress"; + ethertype = "IPv4"; + port_range_max = 0; + port_range_min = 0; + protocol = "tcp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv4-udp-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv4-udp-egress[0]"; + direction = "egress"; + ethertype = "IPv4"; + port_range_max = 0; + port_range_min = 0; + protocol = "udp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv4-udp-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv4-udp-ingress[0]"; + direction = "ingress"; + ethertype = "IPv4"; + port_range_max = 0; + port_range_min = 0; + protocol = "udp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_subnet_v2.cluster_subnet = { + _import_from = "openstack_networking_subnet_v2.cluster_subnet[0]"; + cidr = config.yk8s.infra.subnet_cidr; + dns_nameservers = cfg.dns_nameservers_v4; + ip_version = 4; + name = "${config.yk8s.infra.cluster_name}-network-v4"; + network_id = yk8s-lib.tfRef "openstack_networking_network_v2.cluster_network.id"; + }; + }) + ++ (lib.optional config.yk8s.infra.ipv6_enabled { + resource.openstack_networking_router_interface_v2.cluster_router_iface_v6 = { + _import_from = "openstack_networking_router_interface_v2.cluster_router_iface_v6[0]"; + router_id = yk8s-lib.tfRef "openstack_networking_router_v2.cluster_router.id"; + subnet_id = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_v6_subnet.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-frag-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-frag-egress[0]"; + direction = "egress"; + ethertype = "IPv6"; + protocol = "ipv6-frag"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-frag-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-frag-ingress[0]"; + direction = "ingress"; + ethertype = "IPv6"; + protocol = "ipv6-frag"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-icmp-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-icmp-egress"; + direction = "egress"; + ethertype = "IPv6"; + protocol = "ipv6-icmp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-icmp-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-icmp-ingress[0]"; + direction = "ingress"; + ethertype = "IPv6"; + protocol = "ipv6-icmp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-vrrp-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-vrrp-ingress[0]"; + direction = "ingress"; + ethertype = "IPv6"; + protocol = "vrrp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-vrrp-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-vrrp-egress[0]"; + direction = "egress"; + ethertype = "IPv6"; + protocol = "vrrp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-opts-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-opts-egress[0]"; + direction = "egress"; + ethertype = "IPv6"; + protocol = "ipv6-opts"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-opts-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-opts-ingress[0]"; + direction = "ingress"; + ethertype = "IPv6"; + protocol = "ipv6-opts"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-route-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-route-egress[0]"; + direction = "egress"; + ethertype = "IPv6"; + protocol = "ipv6-route"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-route-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-route-ingress[0]"; + direction = "ingress"; + ethertype = "IPv6"; + protocol = "ipv6-route"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-tcp-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-tcp-egress[0]"; + direction = "egress"; + ethertype = "IPv6"; + port_range_max = 0; + port_range_min = 0; + protocol = "tcp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-tcp-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-tcp-ingress[0]"; + direction = "ingress"; + ethertype = "IPv6"; + port_range_max = 0; + port_range_min = 0; + protocol = "tcp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-udp-egress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-udp-egress[0]"; + direction = "egress"; + ethertype = "IPv6"; + port_range_max = 0; + port_range_min = 0; + protocol = "udp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_secgroup_rule_v2.barndoor-ipv6-udp-ingress = { + _import_from = "openstack_networking_secgroup_rule_v2.barndoor-ipv6-udp-ingress[0]"; + direction = "ingress"; + ethertype = "IPv6"; + port_range_max = 0; + port_range_min = 0; + protocol = "udp"; + security_group_id = yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id"; + }; + + resource.openstack_networking_subnet_v2.cluster_v6_subnet = { + _import_from = "openstack_networking_subnet_v2.cluster_v6_subnet[0]"; + cidr = config.yk8s.infra.subnet_v6_cidr; + ip_version = 6; + ipv6_address_mode = "dhcpv6-stateful"; + ipv6_ra_mode = "dhcpv6-stateful"; + name = "${config.yk8s.infra.cluster_name}-network-v6"; + network_id = yk8s-lib.tfRef "openstack_networking_network_v2.cluster_network.id"; + }; + }); +} diff --git a/nix/yk8s/openstack/20-gateway.nix b/nix/yk8s/openstack/20-gateway.nix new file mode 100644 index 0000000000000000000000000000000000000000..aea26c3f78c6e762f4fbdb977b548405a57bc285 --- /dev/null +++ b/nix/yk8s/openstack/20-gateway.nix @@ -0,0 +1,173 @@ +{ + lib, + yk8s-lib, + config, + ... +}: let + cfg = config.yk8s.openstack; + gateway_nodes = + lib.mapAttrs' ( + name: value: { + name = value.vm_name; + inherit value; + } + ) + (lib.filterAttrs (_: v: v.role == "gateway") cfg.nodes); +in { + yk8s.terraform.modules = + (let + forEachGateway = nodeName: nodeValues: + { + resource."openstack_compute_instance_v2".${nodeName} = { + _import_from = "openstack_compute_instance_v2.gateway[\"${nodeName}\"]"; + availability_zone = nodeValues.az; + config_drive = true; + block_device = lib.optional cfg.create_root_disk_on_volume { + boot_index = 0; + delete_on_termination = true; + destination_type = "volume"; + source_type = "volume"; + uuid = yk8s-lib.tfRef "openstack_blockstorage_volume_v3.${nodeValues.volume_name}.id"; + }; + flavor_id = yk8s-lib.tfRef "data.openstack_compute_flavor_v2.gateway.id"; + image_id = + if cfg.create_root_disk_on_volume + then null + else yk8s-lib.tfRef "data.openstack_images_image_v2.gateway.id"; + key_pair = cfg.keypair; + lifecycle = [{ignore_changes = ["key_pair" "image_id" "config_drive"];}]; + name = nodeName; + network = [{port = yk8s-lib.tfRef "openstack_networking_port_v2.${nodeName}.id";}]; + }; + + resource."openstack_networking_floatingip_associate_v2".${nodeName} = { + _import_from = "openstack_networking_floatingip_associate_v2.gateway[\"${nodeName}\"]"; + depends_on = ["openstack_networking_router_interface_v2.cluster_router_iface"]; + floating_ip = yk8s-lib.tfRef "openstack_networking_floatingip_v2.${nodeName}.address"; + port_id = yk8s-lib.tfRef "openstack_networking_port_v2.${nodeName}.id"; + }; + + resource."openstack_networking_floatingip_v2".${nodeName} = { + _import_from = "openstack_networking_floatingip_v2.gateway[\"${nodeName}\"]"; + description = "Floating IP for gateway '${nodeName}'" + (lib.optionalString (nodeValues.az != null) " in ${nodeValues.az}"); + pool = cfg.public_network; + }; + + resource."openstack_networking_port_v2".${nodeName} = { + _import_from = "openstack_networking_port_v2.gateway[\"${nodeName}\"]"; + allowed_address_pairs = + [{ip_address = yk8s-lib.tfRef "openstack_networking_floatingip_v2.gw_vip_fip.fixed_ip";}] + ++ (lib.optional config.yk8s.infra.ipv4_enabled + {ip_address = "0.0.0.0/0";}) + ++ (lib.optionals config.yk8s.infra.ipv6_enabled [ + {ip_address = "::/0";} + {ip_address = config.yk8s.infra.subnet_v6_cidr;} + ]); + depends_on = ["openstack_networking_floatingip_v2.gw_vip_fip"]; + fixed_ip = + (lib.optional config.yk8s.infra.ipv4_enabled + {subnet_id = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_subnet.id";}) + ++ (lib.optional config.yk8s.infra.ipv6_enabled + {subnet_id = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_v6_subnet.id";}); + lifecycle = [ + { + ignore_changes = [ + # The allowed_address_pairs are subject to change and may get + # (automatically) managed or extended by something else. + # For example, the ch-k8s-lbaas controller manages them to + # allow LBaaS traffic. + # Terraform would reset these settings on each run if we would + # not ignore changes + "allowed_address_pairs" + ]; + } + ]; + name = nodeName; + network_id = yk8s-lib.tfRef "openstack_networking_network_v2.cluster_network.id"; + port_security_enabled = true; + security_group_ids = [ + # Gateway nodes shall accept and forward any traffic hence we use + # the barndoor security group for them. (see #659) + (yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id") + ]; + }; + } + // (lib.optionalAttrs cfg.create_root_disk_on_volume { + resource."openstack_blockstorage_volume_v3".${nodeValues.volume_name} = { + _import_from = "openstack_blockstorage_volume_v3.gateway-volume[\"${nodeName}\"]"; + availability_zone = nodeValues.az; + image_id = yk8s-lib.tfRef "data.openstack_images_image_v2.gateway.id"; + lifecycle = [{ignore_changes = ["image_id"];}]; + name = nodeValues.volume_name; + size = yk8s-lib.tfRef "(data.openstack_compute_flavor_v2.gateway.disk > 0) ? data.openstack_compute_flavor_v2.gateway.disk : ${toString nodeValues.root_disk_size}"; + timeouts = [ + { + create = config.yk8s.terraform.timeout_time; + delete = config.yk8s.terraform.timeout_time; + } + ]; + volume_type = nodeValues.root_disk_volume_type; + }; + }); + in + lib.mapAttrsToList forEachGateway gateway_nodes) + ++ (lib.singleton { + resource.openstack_networking_floatingip_v2.gw_vip_fip = [ + { + depends_on = ["openstack_networking_router_interface_v2.cluster_router_iface"]; + description = "Floating IP associated with the VRRP port"; + pool = cfg.public_network; + port_id = yk8s-lib.tfRef "openstack_networking_port_v2.gw_vip_port.id"; + } + ]; + + resource.openstack_networking_port_v2.gw_vip_port = [ + { + admin_state_up = true; + fixed_ip = + (lib.optional config.yk8s.infra.ipv4_enabled + {subnet_id = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_subnet.id";}) + ++ (lib.optionals config.yk8s.infra.ipv6_enabled [ + {subnet_id = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_v6_subnet.id";} + {subnet_id = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_v6_subnet.id";} + ]); + name = "${config.yk8s.infra.cluster_name}-gateway-vip"; + network_id = yk8s-lib.tfRef "openstack_networking_network_v2.cluster_network.id"; + port_security_enabled = true; + security_group_ids = [ + (yk8s-lib.tfRef "openstack_networking_secgroup_v2.barndoor.id") + ]; + } + ]; + + # output.gateway_ports = [{value = yk8s-lib.tfRef "openstack_networking_port_v2.gateway";}]; + + output = + ( + lib.mapAttrs' (nodeName: _: { + name = "node_${nodeName}"; + value = { + sensitive = true; + value = yk8s-lib.tfRef "openstack_compute_instance_v2.${nodeName}"; + }; + }) + gateway_nodes + ) + // ( + lib.mapAttrs' (nodeName: _: { + name = "floatingip_${nodeName}"; + value.value = yk8s-lib.tfRef "openstack_networking_floatingip_v2.${nodeName}"; + }) + gateway_nodes + ) + // (lib.optionalAttrs config.yk8s.infra.ipv4_enabled { + networking_fixed_ip = [{value = yk8s-lib.tfRef "openstack_networking_port_v2.gw_vip_port.all_fixed_ips[0]";}]; + networking_floating_ip = [{value = yk8s-lib.tfRef "openstack_networking_floatingip_v2.gw_vip_fip.address";}]; + subnet_cidr = [{value = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_subnet.cidr";}]; + }) + // (lib.optionalAttrs config.yk8s.infra.ipv6_enabled { + networking_fixed_ip_v6 = [{value = yk8s-lib.tfRef "openstack_networking_port_v2.gw_vip_port.all_fixed_ips[1]";}]; + subnet_v6_cidr = [{value = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_v6_subnet.cidr";}]; + }); + }); +} diff --git a/nix/yk8s/openstack/40-k8s-nodes.nix b/nix/yk8s/openstack/40-k8s-nodes.nix new file mode 100644 index 0000000000000000000000000000000000000000..f04c1b5494f58e1ddd2604dd17bbbc5294afa3d5 --- /dev/null +++ b/nix/yk8s/openstack/40-k8s-nodes.nix @@ -0,0 +1,148 @@ +{ + lib, + yk8s-lib, + config, + ... +}: let + cfg = config.yk8s.openstack; + nodes = + lib.mapAttrs' ( + _: value: { + name = value.vm_name; + inherit value; + } + ) + (lib.filterAttrs (_: v: v.role != "gateway") cfg.nodes); +in { + yk8s.terraform.modules = lib.singleton { + data.openstack_compute_flavor_v2 = + lib.mapAttrs (_: nodeValues: { + name = nodeValues.flavor; + }) + nodes; + + data.openstack_images_image_v2 = + lib.mapAttrs (_: nodeValues: { + most_recent = true; + name = nodeValues.image; + }) + nodes; + + resource.openstack_compute_servergroup_v2 = builtins.foldl' (acc: policyName: + acc + // { + ${policyName} = { + _import_from = "openstack_compute_servergroup_v2.server_group[\"${policyName}\"]"; + name = policyName; + policies = [ + "anti-affinity" + ]; + }; + }) {} (lib.unique (builtins.filter (v: v != null) (lib.mapAttrsToList (_: v: v.anti_affinity_group) nodes))); + + resource.openstack_blockstorage_volume_v3 = lib.optionalAttrs cfg.create_root_disk_on_volume ( + lib.mapAttrs ( + nodeName: nodeValues: { + _import_from = "openstack_blockstorage_volume_v3.${nodeValues.role}-volume[\"${nodeName}\"]"; + availability_zone = nodeValues.az; + image_id = yk8s-lib.tfRef "data.openstack_images_image_v2.${nodeName}.id"; + lifecycle = [ + { + ignore_changes = [ + "image_id" + ]; + } + ]; + name = nodeValues.volume_name; + size = let + flavorSize = "data.openstack_compute_flavor_v2.${nodeName}.disk"; + in + yk8s-lib.tfRef "(${flavorSize} > 0) ? ${flavorSize} : ${nodeValues.root_disk_size}"; + timeouts = [ + { + create = config.terraform.timeout_time; + delete = config.terraform.timeout_time; + } + ]; + volume_type = nodeValues.root_disk_volume_type; + } + ) + nodes + ); + + resource.openstack_compute_instance_v2 = + lib.mapAttrs ( + nodeName: nodeValues: { + _import_from = "openstack_compute_instance_v2.${nodeValues.role}[\"${nodeName}\"]"; + availability_zone = nodeValues.az; + config_drive = true; + block_device = lib.optional cfg.create_root_disk_on_volume { + boot_index = 0; + delete_on_termination = true; + destination_type = "volume"; + source_type = "volume"; + uuid = yk8s-lib.tfRef "openstack_blockstorage_volume_v3.${nodeName}.id"; + }; + scheduler_hints = lib.optional (nodeValues.anti_affinity_group != null) { + content = [{group = "\${openstack_compute_servergroup_v2.server_group[${nodeValues.anti_affinity_group}].id}";}]; + }; + flavor_id = yk8s-lib.tfRef "data.openstack_compute_flavor_v2.${nodeName}.id"; + image_id = + if cfg.create_root_disk_on_volume + then null + else (yk8s-lib.tfRef "data.openstack_images_image_v2.${nodeName}.id"); + key_pair = yk8s-lib.tfRef "var.keypair"; + lifecycle = [ + { + ignore_changes = [ + "key_pair" + "image_id" + "config_drive" + + # Ignoring 'scheduler_hints' here for existing VMs because otherwise tf would destroy and recreate them. + # The initial distribution for existing clusters must therefore be enforced manually. + "scheduler_hints" + ]; + } + ]; + name = nodeName; + network = [ + { + port = yk8s-lib.tfRef "openstack_networking_port_v2.${nodeName}.id"; + } + ]; + } + ) + nodes; + + resource.openstack_networking_port_v2 = + lib.mapAttrs ( + nodeName: nodeValues: { + _import_from = "openstack_networking_port_v2.${nodeValues.role}[\"${nodeName}\"]"; + fixed_ip = + (lib.optional config.yk8s.infra.ipv4_enabled { + subnet_id = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_subnet.id"; + }) + ++ ( + lib.optional config.yk8s.infra.ipv6_enabled { + subnet_id = yk8s-lib.tfRef "openstack_networking_subnet_v2.cluster_v6_subnet.id"; + } + ); + name = nodeName; + network_id = yk8s-lib.tfRef "openstack_networking_network_v2.cluster_network.id"; + port_security_enabled = false; + } + ) + nodes; + + output = + lib.mapAttrs' (nodeName: _: { + name = "node_${nodeName}"; + value = { + sensitive = true; + value = yk8s-lib.tfRef "openstack_compute_instance_v2.${nodeName}"; + }; + }) + nodes; + }; +} diff --git a/nix/yk8s/openstack/50-object-storage.nix b/nix/yk8s/openstack/50-object-storage.nix new file mode 100644 index 0000000000000000000000000000000000000000..e4141b5d599c5ff3459406edde820f8e55e1c3ee --- /dev/null +++ b/nix/yk8s/openstack/50-object-storage.nix @@ -0,0 +1,15 @@ +{ + lib, + config, + ... +}: let + cfg = config.yk8s.openstack; +in { + yk8s.terraform.modules = lib.singleton { + resource."openstack_objectstorage_container_v1"."thanos_data" = lib.optional cfg.monitoring_manage_thanos_bucket { + _import_from = "openstack_objectstorage_container_v1.thanos_data[0]"; + name = "${config.yk8s.infra.cluster_name}-monitoring-thanos-data"; + force_destroy = cfg.thanos_delete_container; + }; + }; +} diff --git a/nix/yk8s/openstack.nix b/nix/yk8s/openstack/default.nix similarity index 73% rename from nix/yk8s/openstack.nix rename to nix/yk8s/openstack/default.nix index b9a82666cecb5d536c621c6b2eec888593ca9504..29cbe04ed6858dc0fe4b171557eb1c4dd831bcf1 100644 --- a/nix/yk8s/openstack.nix +++ b/nix/yk8s/openstack/default.nix @@ -66,86 +66,21 @@ "nodes" ]; in { + imports = [ + ./00-backend.nix + ./00-provider.nix + ./00-versions.nix + ./00-variables.nix + ./01-discovery.nix + ./10-networking.nix + ./20-gateway.nix + ./40-k8s-nodes.nix + ./50-object-storage.nix + ]; + options.yk8s.openstack = mkTopSection { _docs.order = 1; - _docs.preface = '' - .. note:: - - :ref:`configuration-options.yk8s.openstack.nodes` - allows you to configure - the k8s master and worker servers. - The ``role`` attribute must be used to distinguish both [1]_. - - The amount of gateway nodes can be controlled with - :ref:`configuration-options.yk8s.openstack.gateway_count`. - - .. [1] Caveat: Changing the role of a Terraform node - will completely rebuild the node. - - .. attention:: - - You must configure at least one master node. - - You can add and delete Terraform nodes simply - by adding and removing their entries to/from the config - or tuning :ref:`configuration-options.yk8s.openstack.gateway_count` for gateway nodes. - Consider the following example: - - .. code:: diff - - openstack = { - - - gateway_count = 3; - + gateway_count = 2; # <-- one gateway gets deleted - - nodes = { - worker-0 = { - role = "worker"; - flavor = "M"; - image = "Debian 12 (bookworm)"; - }; - - worker-1 = { # <-- gets deleted - - role = "worker"; - - flavor = "M"; - - }; - worker-2 = { - role = "worker"; - flavor = "L"; - }; - + mon1 = { # <-- gets created - + role = "worker"; - + flavor = "S"; - + image = "Ubuntu 22.04 LTS x64"; - + }; - }; - }; - - The name of an OpenStack node is composed from the following parts: - - - for master/worker nodes: - ``yk8s.infra.cluster_name`` ```` - - - for gateway nodes: - ``yk8s.infra.cluster_name`` ``yk8s.openstack.gateway_defaults.common_name`` ```` - - .. code:: nix - - openstack = { - - cluster_name = "yk8s"; - gateway_count = 1; - #.... - - gateway_defaults.common_name = "gateway-"; - - nodes.master-x.role = "master"; - nodes.worker-a.role = "worker"; - - # yields the following node names: - # - yk8s-gateway-0 - # - yk8s-master-x - # - yk8s-worker-a - ''; + _docs.preface = builtins.readFile ./preface.rst; enabled = mkOption { description = '' @@ -155,6 +90,15 @@ in { default = true; }; + nodes_prefix = yk8s-lib.mkInternalOption { + readOnly = true; + type = lib.types.str; + default = + if config.yk8s.infra.cluster_name == "" + then "" + else "${config.yk8s.infra.cluster_name}-"; + }; + public_network = mkOption { description = '' Name of the Openstack provider network to use @@ -169,7 +113,7 @@ in { Will most of the time be set via the environment variable TF_VAR_keypair ''; type = with types; nullOr openstackKeypairName; - default = null; + default = yk8s-lib.tfRef "var.keypair"; # this will make Terraform read it from TF_VAR_keypair }; azs = mkOption { @@ -266,6 +210,8 @@ in { You may also specify those attributes or a subset of them using :ref:`yk8s.openstack.{master,worker}_defaults `. + + Gateways are created automatically, and should not be explicitly added here. ''; type = types.attrsOf (types.submodule { options = { @@ -273,6 +219,7 @@ in { type = types.enum [ "master" "worker" + "gateway" ]; }; image = mkOption { @@ -303,9 +250,40 @@ in { type = with types; nullOr openstackServerGroupName; default = null; }; + vm_name = mkInternalOption { + type = types.nonEmptyStr; + }; + volume_name = mkInternalOption { + type = types.nonEmptyStr; + }; }; }); default = {}; + apply = lib.mapAttrs ( + # fill all null values with defaults + nodeName: nodeValues: (lib.foldlAttrs (acc: k: v: + acc + // lib.optionalAttrs ((!builtins.hasAttr k acc) || v != null) { + ${k} = v; + }) ( + lib.getAttr nodeValues.role + { + "master" = cfg.master_defaults; + "worker" = cfg.worker_defaults; + "gateway" = cfg.gateway_defaults; + } + ) + ( + nodeValues + // rec { + vm_name = "${cfg.nodes_prefix}${nodeName}"; + volume_name = + if nodeValues.role == "gateway" + then "${vm_name}-volume" + else "${config.yk8s.infra.cluster_name}-${nodeValues.role}-volume-${nodeName}"; + } + )) + ); }; network_name = mkOption { @@ -350,9 +328,35 @@ in { ''; }; config.yk8s = lib.mkMerge [ + { + _targets.ansible.inventory_packages = [ + (mkGroupVarsFile { + inherit cfg; + inventory_path = "all/openstack.yaml"; + ansible_prefix = "openstack_"; + only_if_enabled = true; + transformations = [(c: builtins.removeAttrs c nonAnsibleOptions)]; + }) + ]; + } (lib.mkIf cfg.enabled { terraform.enabled = true; + openstack.nodes = builtins.foldl' (acc: idx: let + node_name = "${cfg.gateway_defaults.common_name}${toString idx}"; + in + acc + // { + "${node_name}" = { + role = "gateway"; + az = + if cfg.spread_gateways_across_azs + then builtins.elemAt cfg.azs (lib.mod idx (builtins.length cfg.azs)) + else null; + }; + }) + {} (lib.range 0 (cfg.gateway_count - 1)); + assertions = let inherit (builtins) all length filter attrValues; in [ @@ -408,17 +412,59 @@ in { message = "config.yk8s.openstack.network_mtu: must be at least 1280 Bytes to support IPv6."; } ]; + + infra = { + networking_floating_ip = config.yk8s.terraform.outputs.networking_floating_ip.value; + networking_fixed_ip = config.yk8s.terraform.outputs.networking_fixed_ip.value or null; + networking_fixed_ip_v6 = config.yk8s.terraform.outputs.networking_fixed_ip_v6.value or null; + ansible_hosts = let + isGateway = nodeName: cfg.nodes.${nodeName}.role == "gateway"; + globalConfig = { + all.vars = { + }; + }; + nodeConfigs = + lib.mapAttrs ( + nodeName: nodeValues: let + vmValues = config.yk8s.terraform.outputs."node_${nodeValues.vm_name}".value; + networkValues = builtins.head vmValues.network; + fipValues = config.yk8s.terraform.outputs."floatingip_${nodeValues.vm_name}".value; + in + rec { + ansible_host = + if isGateway nodeName + then fipValues.address + else local_ipv4_address; + port_id = networkValues.port; + local_ipv4_address = networkValues.fixed_ip_v4; + } + // lib.optionalAttrs config.yk8s.infra.ipv6_enabled { + local_ipv6_address = lib.removePrefix "[" (lib.removeSuffix "]" networkValues.fixed_ip_v6); + } + ) + cfg.nodes; + in + lib.foldlAttrs ( + acc: nodeName: nodeConfig: let + path = + (lib.getAttr cfg.nodes.${nodeName}.role + { + "gateway" = ["gateways"]; + "master" = ["masters"]; + "worker" = ["workers"]; + }) + ++ ["hosts" "${cfg.nodes.${nodeName}.vm_name}"]; + in + lib.recursiveUpdate acc (lib.setAttrByPath path nodeConfig) + ) + globalConfig + nodeConfigs; + }; + + ch-k8s-lbaas = { + subnet_id = config.yk8s.terraform.outputs.subnet_id.value; + floating_ip_network_id = config.yk8s.terraform.outputs.floating_ip_network_id.value; + }; }) - { - _inventory_packages = [ - (mkGroupVarsFile { - inherit cfg; - inventory_path = "all/openstack.yaml"; - ansible_prefix = "openstack_"; - only_if_enabled = true; - transformations = [(c: builtins.removeAttrs c nonAnsibleOptions)]; - }) - ]; - } ]; } diff --git a/nix/yk8s/openstack/preface.rst b/nix/yk8s/openstack/preface.rst new file mode 100644 index 0000000000000000000000000000000000000000..423b12df6e34d266cb014443b359c195856dab23 --- /dev/null +++ b/nix/yk8s/openstack/preface.rst @@ -0,0 +1,76 @@ +.. note:: + + :ref:`configuration-options.yk8s.openstack.nodes` + allows you to configure + the k8s master and worker servers. + The ``role`` attribute must be used to distinguish both [1]_. + + The amount of gateway nodes can be controlled with + :ref:`configuration-options.yk8s.openstack.gateway_count`. + +.. [1] Caveat: Changing the role of a Terraform node + will completely rebuild the node. + +.. attention:: + + You must configure at least one master node. + +You can add and delete Terraform nodes simply +by adding and removing their entries to/from the config +or tuning :ref:`configuration-options.yk8s.openstack.gateway_count` for gateway nodes. +Consider the following example: + +.. code:: diff + + openstack = { + + - gateway_count = 3; + + gateway_count = 2; # <-- one gateway gets deleted + + nodes = { + worker-0 = { + role = "worker"; + flavor = "M"; + image = "Debian 12 (bookworm)"; + }; + - worker-1 = { # <-- gets deleted + - role = "worker"; + - flavor = "M"; + - }; + worker-2 = { + role = "worker"; + flavor = "L"; + }; + + mon1 = { # <-- gets created + + role = "worker"; + + flavor = "S"; + + image = "Ubuntu 22.04 LTS x64"; + + }; + }; + }; + +The name of an OpenStack node is composed from the following parts: + +- for master/worker nodes: +``yk8s.infra.cluster_name`` ```` + +- for gateway nodes: +``yk8s.infra.cluster_name`` ``yk8s.openstack.gateway_defaults.common_name`` ```` + +.. code:: nix + + openstack = { + + cluster_name = "yk8s"; + gateway_count = 1; + #.... + + gateway_defaults.common_name = "gateway-"; + + nodes.master-x.role = "master"; + nodes.worker-a.role = "worker"; + + # yields the following node names: + # - yk8s-gateway-0 + # - yk8s-master-x + # - yk8s-worker-a diff --git a/nix/yk8s/terraform.nix b/nix/yk8s/terraform.nix index 3bdb1c8b7e74e8d12363f02419d1284f03d4ef9a..b7bd722cd3a982193a6f4b4e5309933b59736b24 100644 --- a/nix/yk8s/terraform.nix +++ b/nix/yk8s/terraform.nix @@ -2,7 +2,9 @@ config, lib, yk8s-lib, + terranix-lib, pkgs, + system, ... }: let cfg = config.yk8s.terraform; @@ -138,7 +140,7 @@ in { gitlab_backend = mkEnableOption '' GitLab-managed Terraform backend If true, the Terraform state will be stored inside the provided gitlab project. - If set, the environment `TF_HTTP_USERNAME` and `TF_HTTP_PASSWO = mkOptionD` + If set, the environment `TF_HTTP_USERNAME` and `TF_HTTP_PASSWORD` must be configured in a separate file `~/.config/yaook-k8s/env`. ''; @@ -157,14 +159,6 @@ in { ''; type = with types; nullOr gitlabProjectId; default = null; - apply = v: - if - cfg.gitlab_backend - && v == null - then - throw - "config.yk8s.terraform.gitlab_project_id: must be set because config.yk8s.terraform.gitlab_backend=true" - else v; }; gitlab_state_name = mkOption { @@ -175,43 +169,108 @@ in { default = null; example = "tf-state"; }; + + gitlab_backend_address = mkInternalOption { + readOnly = true; + type = with types; nullOr nonEmptyStr; + }; + + modules = mkOption { + type = with types; listOf anything; + default = []; + }; + + outputs = mkInternalOption { + readOnly = true; + type = types.attrs; + default = let + tfOutputsPath = "terraform/outputs.json"; + tfOutputsFullPath = "${config.yk8s.state_directory}/${tfOutputsPath}"; + in + if config.yk8s.state_directory != null && builtins.pathExists tfOutputsFullPath + then lib.importJSON tfOutputsFullPath + else throw "${tfOutputsPath} does not exist yet. Terraform stage needs to be run first."; + }; + + migrations = mkInternalOption { + # TODO: allow different types of migration (mv, rm...) + type = types.listOf (types.submodule { + options = { + from = mkOption { + type = types.nonEmptyStr; + }; + to = mkOption { + type = types.nonEmptyStr; + }; + }; + }); + }; }; - config.yk8s = { - _inventory_packages = - [ - (mkGroupVarsFile { - cfg = lib.attrsets.getAttrs ["enabled"] cfg; - inventory_path = "all/terraform.yaml"; - }) - ] - ++ lib.optionals cfg.enabled ( - let - linkTfstateIfExists = source: target: - if config.yk8s.state_directory != null && builtins.pathExists "${config.yk8s.state_directory}/${source}" - then [(linkToPath "${config.yk8s.state_directory}/${source}" target)] - else - builtins.trace "INFO: ${config.yk8s._state_base_path}/${source} does not yet exist. Terraform stage needs to be run first." - []; - in - (linkTfstateIfExists "terraform/rendered/hosts" "hosts") - ++ (linkTfstateIfExists "terraform/rendered/terraform_networking-trampoline.yaml" "group_vars/all/terraform_networking-trampoline.yaml") - ++ (linkTfstateIfExists "terraform/rendered/terraform_networking.yaml" "group_vars/all/terraform_networking.yaml") - ); - _state_packages = - lib.optional cfg.enabled - ( - let - filteredTerraformCfg = yk8s-lib.removeAttrsByPath config.yk8s.terraform [["enabled"]]; - filteredInfraCfg = lib.attrsets.getAttrs infraTerraformOptions config.yk8s.infra; - filteredOpenstackCfg = lib.attrsets.getAttrs openstackTerraformOptions config.yk8s.openstack; - mergedCfg = - builtins.foldl' (acc: e: lib.attrsets.recursiveUpdate acc (removeObsoleteOptions e)) {} - [filteredTerraformCfg filteredInfraCfg filteredOpenstackCfg]; - transformations = [filterInternal filterNull]; - varsFile = mkJson "tfvars.json" (pipe mergedCfg transformations); - in (pkgs.runCommandLocal "tfvars.json" {} '' - install -m 644 -D ${varsFile} $out/${tfvars_file_path} - '') + + config.yk8s = let + all_gitlab_vars = ["gitlab_base_url" "gitlab_project_id" "gitlab_state_name"]; + all_gitlab_vars_are_set = lib.all (v: v != null) (builtins.attrValues (lib.getAttrs all_gitlab_vars cfg)); + all_gitlab_vars_are_unset = lib.all (v: v == null) (builtins.attrValues (lib.getAttrs all_gitlab_vars cfg)); + in + lib.mkIf cfg.enabled { + terraform.gitlab_backend_address = + if all_gitlab_vars_are_set + then "${cfg.gitlab_base_url}/api/v4/projects/${cfg.gitlab_project_id}/terraform/state/${cfg.gitlab_state_name}" + else null; + + terraform.migrations = let + getMigrations = path: value: + lib.optionals (builtins.isAttrs value) ( + if value ? "_import_from" + then + lib.singleton { + from = value._import_from; + to = lib.strings.concatStringsSep "." path; + } + else lib.foldlAttrs (acc: k: v: acc ++ getMigrations (path ++ [k]) v) [] value + ); + in + builtins.foldl' (acc: mod: acc ++ (getMigrations [] (mod.resource or {}))) [] cfg.modules; + + assertions = [ + { + assertion = cfg.gitlab_backend -> all_gitlab_vars_are_set; + message = "[yk8s.terraform] gitlab_backend=true' but GitLab variables are not (completely) provided. Please set all of ${lib.concatStringsSep " " all_gitlab_vars}"; + } + { + assertion = all_gitlab_vars_are_set || all_gitlab_vars_are_unset; + message = '' + [yk8s.terraform] gitlab_backend=false but some GitLab variables are provided. + (1) If you want to migrate the Terraform backend method from 'http' to 'local', + you should provide all the GitLab variables + (2) If you want to init a cluster with local backend, + make sure that all all of ${lib.concatStringsSep " " all_gitlab_vars} are unset. + ''; + } + ]; + + _targets.terraform.state_packages = let + tfConfig = terranix-lib.terranixConfiguration { + inherit system; + modules = builtins.map (lib.filterAttrsRecursive (k: _: k != "_import_from")) cfg.modules; + }; + in [ + (yk8s-lib.linkToPath tfConfig "terraform/config.tf.json") + (yk8s-lib.mkJsonAtPath "terraform/gitlab.conf.json" (lib.getAttrs (all_gitlab_vars ++ ["gitlab_backend"]) cfg)) + ]; + }; + config.packages.tf-state-migrations = builtins.seq (yk8s-lib.baseSystemAssertWarn config.yk8s) (pkgs.writeScript "tf-state-migrations" ( + lib.strings.concatMapStringsSep "\n" (v: "run terraform state mv -state \"$tf_statefile_temp\" '${v.from}' '${v.to}'") cfg.migrations + )); + config.packages.tf-state-migrations-undo = pkgs.writeScript "tf-state-migrations" ( + lib.strings.concatMapStringsSep "\n" (v: "run terraform state mv -state \"$tf_statefile_temp\" '${v.to}' '${v.from}'") cfg.migrations + ); + config.packages.tf-state-migrations-json = yk8s-lib.mkJson "tf-state-migrations.json" { + migration.state.yk8s = { + dir = yk8s-lib.tfRef "env.terraform_state_dir"; + actions = ( + map (v: "mv '${v.from}' '${v.to}'") cfg.migrations ); + }; }; } diff --git a/nix/yk8s/testing.nix b/nix/yk8s/testing.nix index ce50cc9c808e0434b001d76c4ba9c4259092a9a4..ace3963448da00006f7618fe49728ad5e250f999 100644 --- a/nix/yk8s/testing.nix +++ b/nix/yk8s/testing.nix @@ -48,7 +48,7 @@ in { message = "config.yk8s.testing.nodes: nodes [${concatStringsSep ", " nonExistentNodes}] don't exist. Note that full hostnames including the prefix '${config.yk8s.infra.cluster_name}-' must be supplied."; } ]; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ ( mkGroupVarsFile { inherit cfg; diff --git a/nix/yk8s/vault.nix b/nix/yk8s/vault.nix index 83ddbdda72daef17bfa4ec16580e0e153699349f..d78513174c235782cc3d3040c051f981e9fea6e9 100644 --- a/nix/yk8s/vault.nix +++ b/nix/yk8s/vault.nix @@ -6,7 +6,7 @@ }: let cfg = config.yk8s.vault; inherit (lib) mkOption types; - inherit (yk8s-lib) mkTopSection mkGroupVarsFile; + inherit (yk8s-lib) mkTopSection mkGroupVarsFile mkYamlAtPath; inherit (yk8s-lib.types) vaultChildNamespaceNameSegment @@ -40,11 +40,28 @@ in { default = "yaook/nodes"; }; }; - config.yk8s._inventory_packages = [ + config.yk8s._targets.ansible.inventory_packages = [ (mkGroupVarsFile { inherit cfg; ansible_prefix = "vault_"; inventory_path = "all/vault-backend.yaml"; }) ]; + config.yk8s._targets.vault = { + inventory_subdir = "vault"; + inventory_packages = [ + ( + mkYamlAtPath "main.yaml" { + wg_usage = config.yk8s.wireguard.enabled; + k8s_controller_manager_enable_signing_requests = config.yk8s.kubernetes.controller_manager.enable_signing_requests; + vault_cluster_name = cfg.cluster_name; + thanos_enabled = config.yk8s.k8s-service-layer.prometheus.use_thanos; + manage_thanos_bucket = config.yk8s.k8s-service-layer.prometheus.manage_thanos_bucket; + thanos_config_file = config.yk8s.k8s-service-layer.prometheus.thanos_objectstorage_config_file; + vault_backup_s3_enabled = config.yk8s.k8s-service-layer.vault.enable_backups; + vault_backup_s3_config_file = config.yk8s.k8s-service-layer.vault.s3_config_file; + } + ) + ]; + }; } diff --git a/terraform/.gitignore b/terraform/.gitignore deleted file mode 100644 index 9159dd79c64506b1e2a59e96ee83e3db313059d1..0000000000000000000000000000000000000000 --- a/terraform/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/.terraform -.terraform.lock.hcl -backend_override.tf diff --git a/terraform/00-provider.tf b/terraform/00-provider.tf deleted file mode 100644 index 708ea0afe4f11f581c071562020f071abc37eb75..0000000000000000000000000000000000000000 --- a/terraform/00-provider.tf +++ /dev/null @@ -1,3 +0,0 @@ -provider "openstack" { - -} diff --git a/terraform/00-variables.tf b/terraform/00-variables.tf deleted file mode 100644 index 637a2278c19a43c2bd84d2d8197a6722671c187a..0000000000000000000000000000000000000000 --- a/terraform/00-variables.tf +++ /dev/null @@ -1,144 +0,0 @@ -variable "cluster_name" { - type = string -} - -variable "subnet_cidr" { - type = string - # The following basically makes the variable optional - nullable = true - default = null -} - -variable "subnet_v6_cidr" { - type = string - # The following basically makes the variable optional - nullable = true - default = null -} - -variable "ipv6_enabled" { - type = bool - default = false -} - -variable "ipv4_enabled" { - type = bool -} - -variable "public_network" { - type = string -} - -variable "keypair" { - type = string -} - -variable "azs" { - type = set(string) -} - -variable "thanos_delete_container" { - type = bool -} - -// Setting this to false is useful in CI environments if the Cloud Is Full. -variable "spread_gateways_across_azs" { - type = bool -} - -variable "create_root_disk_on_volume" { - type = bool -} - -variable "timeout_time" { - type = string -} - -variable "network_mtu" { - type = number -} - -variable "dns_nameservers_v4" { - type = list(string) -} - -variable "monitoring_manage_thanos_bucket" { - type = bool -} - -# tflint-ignore: terraform_unused_declarations -variable "gitlab_backend" { - type = bool - default = false -} - -# tflint-ignore: terraform_unused_declarations -variable "gitlab_base_url" { - type = string - default = "" -} - -# tflint-ignore: terraform_unused_declarations -variable "gitlab_project_id" { - type = string - default = "" -} - -# tflint-ignore: terraform_unused_declarations -variable "gitlab_state_name" { - type = string - default = "" -} - - -variable "gateway_count" { - type = number -} - -variable "gateway_defaults" { - type = object({ # --- template spec --- - common_name = string - image = string - flavor = string - root_disk_size = number - root_disk_volume_type = optional(string) - }) -} - -variable "master_defaults" { - type = object({ # --- template spec --- - image = string - flavor = string - root_disk_size = number - root_disk_volume_type = optional(string) - }) -} - -variable "worker_defaults" { - type = object({ # --- template spec --- - image = string - flavor = string - root_disk_size = number - root_disk_volume_type = optional(string) - anti_affinity_group = optional(string) - }) -} - - -variable "nodes" { - type = map( - object({ - role = string # one of: 'master', 'worker' - image = optional(string) - flavor = optional(string) - az = optional(string) - root_disk_size = optional(number) - root_disk_volume_type = optional(string) - anti_affinity_group = optional(string) - }) - ) -} - -locals { - nodes_prefix = (var.cluster_name == "" ? "" : "${var.cluster_name}-") -} diff --git a/terraform/01-discovery.tf b/terraform/01-discovery.tf deleted file mode 100644 index 29a28bb92ec25f2f1744687fda8ce6716eec4bbb..0000000000000000000000000000000000000000 --- a/terraform/01-discovery.tf +++ /dev/null @@ -1,12 +0,0 @@ -data "openstack_networking_network_v2" "public_network" { - name = var.public_network -} - -data "openstack_compute_flavor_v2" "gateway" { - name = var.gateway_defaults.flavor -} - -data "openstack_images_image_v2" "gateway" { - name = var.gateway_defaults.image - most_recent = true -} diff --git a/terraform/10-networking.tf b/terraform/10-networking.tf deleted file mode 100644 index 990631952f05341d4f1c743c4df97d6c932831ab..0000000000000000000000000000000000000000 --- a/terraform/10-networking.tf +++ /dev/null @@ -1,272 +0,0 @@ -resource "openstack_networking_network_v2" "cluster_network" { - name = "${var.cluster_name}-network" - admin_state_up = true - mtu = var.network_mtu - lifecycle { - ignore_changes = [mtu] - } -} - -resource "openstack_networking_subnet_v2" "cluster_subnet" { - # Create only if ipv4 support is enabled - count = var.ipv4_enabled ? 1 : 0 - - name = "${var.cluster_name}-network-v4" - network_id = openstack_networking_network_v2.cluster_network.id - cidr = var.subnet_cidr - ip_version = 4 - dns_nameservers = var.dns_nameservers_v4 -} - -resource "openstack_networking_subnet_v2" "cluster_v6_subnet" { - # Create only if ipv6 support is enabled - count = var.ipv6_enabled ? 1 : 0 - - name = "${var.cluster_name}-network-v6" - network_id = openstack_networking_network_v2.cluster_network.id - cidr = var.subnet_v6_cidr - ip_version = 6 - ipv6_address_mode = "dhcpv6-stateful" - ipv6_ra_mode = "dhcpv6-stateful" -} - -resource "openstack_networking_router_v2" "cluster_router" { - name = "${var.cluster_name}-router" - admin_state_up = true - external_network_id = data.openstack_networking_network_v2.public_network.id -} - -resource "openstack_networking_router_interface_v2" "cluster_router_iface" { - # Create only if ipv4 support is enabled - count = var.ipv4_enabled ? 1 : 0 - - router_id = openstack_networking_router_v2.cluster_router.id - subnet_id = openstack_networking_subnet_v2.cluster_subnet[0].id -} - -resource "openstack_networking_router_interface_v2" "cluster_router_iface_v6" { - # Create only if ipv6 support is enabled - count = var.ipv6_enabled ? 1 : 0 - - router_id = openstack_networking_router_v2.cluster_router.id - subnet_id = openstack_networking_subnet_v2.cluster_v6_subnet[0].id -} - -resource "openstack_networking_secgroup_v2" "barndoor" { - name = "barndoor" - description = "A barndoor wide open. Allows ANY kind of traffic." -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv4-tcp-ingress" { - count = var.ipv4_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv4" - protocol = "tcp" - port_range_min = 0 - port_range_max = 0 - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv4-tcp-egress" { - count = var.ipv4_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv4" - protocol = "tcp" - port_range_min = 0 - port_range_max = 0 - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv4-udp-ingress" { - count = var.ipv4_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv4" - protocol = "udp" - port_range_min = 0 - port_range_max = 0 - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv4-udp-egress" { - count = var.ipv4_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv4" - protocol = "udp" - port_range_min = 0 - port_range_max = 0 - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv4-icmp-ingress" { - count = var.ipv4_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv4" - protocol = "icmp" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv4-icmp-egress" { - count = var.ipv4_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv4" - protocol = "icmp" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv4-vrrp-ingress" { - count = var.ipv4_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv4" - protocol = "vrrp" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv4-vrrp-egress" { - count = var.ipv4_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv4" - protocol = "vrrp" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-tcp-ingress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv6" - protocol = "tcp" - port_range_min = 0 - port_range_max = 0 - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-tcp-egress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv6" - protocol = "tcp" - port_range_min = 0 - port_range_max = 0 - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-udp-ingress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv6" - protocol = "udp" - port_range_min = 0 - port_range_max = 0 - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-udp-egress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv6" - protocol = "udp" - port_range_min = 0 - port_range_max = 0 - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-icmp-ingress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv6" - protocol = "ipv6-icmp" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-icmp-egress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv6" - protocol = "ipv6-icmp" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-route-ingress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv6" - protocol = "ipv6-route" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-route-egress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv6" - protocol = "ipv6-route" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-frag-ingress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv6" - protocol = "ipv6-frag" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-frag-egress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv6" - protocol = "ipv6-frag" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-opts-ingress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv6" - protocol = "ipv6-opts" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-opts-egress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv6" - protocol = "ipv6-opts" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-vrrp-ingress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "ingress" - ethertype = "IPv6" - protocol = "vrrp" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} - -resource "openstack_networking_secgroup_rule_v2" "barndoor-ipv6-vrrp-egress" { - count = var.ipv6_enabled ? 1 : 0 - - direction = "egress" - ethertype = "IPv6" - protocol = "vrrp" - security_group_id = openstack_networking_secgroup_v2.barndoor.id -} diff --git a/terraform/20-gateway.tf b/terraform/20-gateway.tf deleted file mode 100644 index c010c1007d9046a064d870ae5fee469c1d92c0cd..0000000000000000000000000000000000000000 --- a/terraform/20-gateway.tf +++ /dev/null @@ -1,200 +0,0 @@ -locals { - gateway_nodes = { - for idx in range(var.gateway_count) : - "${local.nodes_prefix}${var.gateway_defaults.common_name}${idx}" => { - image = var.gateway_defaults.image - flavor = var.gateway_defaults.flavor - az = var.spread_gateways_across_azs ? tolist(var.azs)[idx % length(var.azs)] : null - volume_name = "${local.nodes_prefix}${var.gateway_defaults.common_name}${idx}-volume" - root_disk_size = var.gateway_defaults.root_disk_size - root_disk_volume_type = var.gateway_defaults.root_disk_volume_type - } - } -} - -resource "openstack_networking_port_v2" "gw_vip_port" { - name = "${var.cluster_name}-gateway-vip" - admin_state_up = true - network_id = openstack_networking_network_v2.cluster_network.id - port_security_enabled = true - - dynamic "fixed_ip" { - for_each = var.ipv4_enabled ? [1] : [] - content { - subnet_id = openstack_networking_subnet_v2.cluster_subnet[0].id - } - } - - dynamic "fixed_ip" { - for_each = var.ipv6_enabled ? [1] : [] - content { - subnet_id = openstack_networking_subnet_v2.cluster_v6_subnet[0].id - } - } - - dynamic "fixed_ip" { - for_each = var.ipv6_enabled ? [1] : [] - content { - subnet_id = openstack_networking_subnet_v2.cluster_v6_subnet[0].id - } - } - - security_group_ids = [ - # NOTE: Openstack OVN usage requires port security to be enabled. - # Gateway nodes shall accept and forward any traffic hence we use - # the barndoor security group for them. (see #659) - openstack_networking_secgroup_v2.barndoor.id, - ] -} - -resource "openstack_networking_floatingip_v2" "gw_vip_fip" { - pool = var.public_network - description = "Floating IP associated with the VRRP port" - port_id = openstack_networking_port_v2.gw_vip_port.id - - depends_on = [ - openstack_networking_router_interface_v2.cluster_router_iface[0] - ] -} - - -resource "openstack_networking_port_v2" "gateway" { - for_each = local.gateway_nodes - name = each.key - - network_id = openstack_networking_network_v2.cluster_network.id - port_security_enabled = true - - dynamic "fixed_ip" { - for_each = var.ipv4_enabled ? [1] : [] - content { - subnet_id = openstack_networking_subnet_v2.cluster_subnet[0].id - } - } - - dynamic "fixed_ip" { - for_each = var.ipv6_enabled ? [1] : [] - content { - subnet_id = openstack_networking_subnet_v2.cluster_v6_subnet[0].id - } - } - - allowed_address_pairs { - ip_address = openstack_networking_floatingip_v2.gw_vip_fip.fixed_ip - } - - dynamic "allowed_address_pairs" { - for_each = var.ipv4_enabled ? [1] : [] - content { - ip_address = "0.0.0.0/0" - } - } - - dynamic "allowed_address_pairs" { - for_each = var.ipv6_enabled ? [1] : [] - content { - ip_address = "::/0" - } - } - - dynamic "allowed_address_pairs" { - for_each = var.ipv6_enabled ? [1] : [] - content { - ip_address = var.subnet_v6_cidr - } - } - - depends_on = [ - openstack_networking_floatingip_v2.gw_vip_fip - ] - - security_group_ids = [ - openstack_networking_secgroup_v2.barndoor.id, - ] - - lifecycle { - ignore_changes = [ - # The allowed_address_pairs are subject to change and may get - # (automatically) managed or extended by something else. - # For example, the ch-k8s-lbaas controller manages them to - # allow LBaaS traffic. - # Terraform would reset these settings on each run if we would - # not ignore changes - allowed_address_pairs - ] - } -} - -resource "openstack_blockstorage_volume_v3" "gateway-volume" { - for_each = var.create_root_disk_on_volume == true ? local.gateway_nodes : {} - name = each.value.volume_name - size = (data.openstack_compute_flavor_v2.gateway.disk > 0) ? data.openstack_compute_flavor_v2.gateway.disk : each.value.root_disk_size - image_id = data.openstack_images_image_v2.gateway.id - volume_type = each.value.root_disk_volume_type - availability_zone = each.value.az - - timeouts { - create = var.timeout_time - delete = var.timeout_time - } - - lifecycle { - ignore_changes = [image_id] - } -} - -resource "openstack_compute_instance_v2" "gateway" { - for_each = local.gateway_nodes - - name = each.key - flavor_id = data.openstack_compute_flavor_v2.gateway.id - image_id = var.create_root_disk_on_volume == false ? data.openstack_images_image_v2.gateway.id : null - key_pair = var.keypair - availability_zone = each.value.az - config_drive = true - - dynamic block_device { - # Using "for_each" for check the conditional "create_root_disk_on_volume". It's not working as a loop. "dummy" should make this just more visible. - for_each = var.create_root_disk_on_volume == true ? ["dummy"] : [] - content { - uuid = openstack_blockstorage_volume_v3.gateway-volume[each.key].id - source_type = "volume" - boot_index = 0 - destination_type = "volume" - delete_on_termination = true - } - } - - network { - port = openstack_networking_port_v2.gateway[each.key].id - } - lifecycle { - ignore_changes = [key_pair, image_id, config_drive] - } -} - -resource "openstack_networking_floatingip_v2" "gateway" { - for_each = local.gateway_nodes - description = "Floating IP for gateway '${each.key}'${each.value.az != null ? " in ${each.value.az}" : ""}" - pool = var.public_network -} - -resource "openstack_networking_floatingip_associate_v2" "gateway" { - for_each = openstack_compute_instance_v2.gateway - - floating_ip = openstack_networking_floatingip_v2.gateway[each.key].address - port_id = openstack_networking_port_v2.gateway[each.key].id - - depends_on = [ - openstack_networking_router_interface_v2.cluster_router_iface[0] - ] -} - -data "template_file" "trampoline_gateways" { - template = file("${path.module}/templates/trampoline_gateways.tpl") - vars = { - networking_fixed_ip = try(jsonencode(openstack_networking_port_v2.gw_vip_port.all_fixed_ips[0]), "null"), - networking_fixed_ip_v6 = try(jsonencode(openstack_networking_port_v2.gw_vip_port.all_fixed_ips[1]), "null"), - networking_floating_ip = openstack_networking_floatingip_v2.gw_vip_fip.address, - } -} diff --git a/terraform/30-master-nodes.tf b/terraform/30-master-nodes.tf deleted file mode 100644 index 821a05776339bd9580933da694f9fd0d208f3b3b..0000000000000000000000000000000000000000 --- a/terraform/30-master-nodes.tf +++ /dev/null @@ -1,99 +0,0 @@ -locals { - # NOTE: coalesce() is used to provide non-null default values from the templates - master_nodes = { - for name, values in var.nodes : - "${local.nodes_prefix}${name}" => { - image = coalesce(values.image, var.master_defaults.image) - flavor = coalesce(values.flavor, var.master_defaults.flavor) - az = values.az # default: null - volume_name = "${var.cluster_name}-master-volume-${name}" - root_disk_size = coalesce(values.root_disk_size, var.master_defaults.root_disk_size) - root_disk_volume_type = values.root_disk_volume_type != null ? values.root_disk_volume_type : var.master_defaults.root_disk_volume_type - } if values.role == "master" - } -} - -resource "openstack_networking_port_v2" "master" { - for_each = local.master_nodes - name = each.key - - network_id = openstack_networking_network_v2.cluster_network.id - - dynamic "fixed_ip" { - for_each = var.ipv4_enabled ? [1] : [] - content { - subnet_id = openstack_networking_subnet_v2.cluster_subnet[0].id - } - } - - dynamic "fixed_ip" { - for_each = var.ipv6_enabled ? [1] : [] - content { - subnet_id = openstack_networking_subnet_v2.cluster_v6_subnet[0].id - } - } - - port_security_enabled = false -} - -data "openstack_compute_flavor_v2" "master" { - for_each = local.master_nodes - name = each.value.flavor -} - -data "openstack_images_image_v2" "master" { - for_each = local.master_nodes - name = each.value.image - most_recent = true -} - -resource "openstack_blockstorage_volume_v3" "master-volume" { - for_each = var.create_root_disk_on_volume == true ? local.master_nodes : {} - - name = each.value.volume_name - size = (data.openstack_compute_flavor_v2.master[each.key].disk > 0) ? data.openstack_compute_flavor_v2.master[each.key].disk : each.value.root_disk_size - image_id = data.openstack_images_image_v2.master[each.key].id - volume_type = each.value.root_disk_volume_type - availability_zone = each.value.az - - timeouts { - create = var.timeout_time - delete = var.timeout_time - } - - lifecycle { - ignore_changes = [image_id] - } -} - -resource "openstack_compute_instance_v2" "master" { - for_each = local.master_nodes - name = each.key - - availability_zone = each.value.az - config_drive = true - flavor_id = data.openstack_compute_flavor_v2.master[each.key].id - image_id = var.create_root_disk_on_volume == false ? data.openstack_images_image_v2.master[each.key].id : null - key_pair = var.keypair - - dynamic block_device { - # Abusing 'for_each' as a conditional - # It's not working as a loop. The outer `each.key` is "passed" into the inner `for_each` - for_each = var.create_root_disk_on_volume == true ? [each.key] : [] - content { - uuid = openstack_blockstorage_volume_v3.master-volume[each.key].id - source_type = "volume" - boot_index = 0 - destination_type = "volume" - delete_on_termination = true - } - } - - network { - port = openstack_networking_port_v2.master[each.key].id - } - - lifecycle { - ignore_changes = [key_pair, image_id, config_drive] - } -} diff --git a/terraform/40-worker-nodes.tf b/terraform/40-worker-nodes.tf deleted file mode 100644 index 2c96cc6097571c98e0cdc8ad059aa31814bf9925..0000000000000000000000000000000000000000 --- a/terraform/40-worker-nodes.tf +++ /dev/null @@ -1,119 +0,0 @@ -locals { - # NOTE: coalesce() is used to provide non-null default values from the templates - worker_nodes = { - for name, values in var.nodes : - "${local.nodes_prefix}${name}" => { - image = coalesce(values.image, var.worker_defaults.image) - flavor = coalesce(values.flavor, var.worker_defaults.flavor) - az = values.az # default: null - volume_name = "${var.cluster_name}-worker-volume-${name}" - root_disk_size = coalesce(values.root_disk_size, var.worker_defaults.root_disk_size) - root_disk_volume_type = values.root_disk_volume_type != null ? values.root_disk_volume_type : var.worker_defaults.root_disk_volume_type - anti_affinity_group = values.anti_affinity_group != null ? values.anti_affinity_group : var.worker_defaults.anti_affinity_group - } if values.role == "worker" - } -} - -resource "openstack_networking_port_v2" "worker" { - for_each = local.worker_nodes - name = each.key - - network_id = openstack_networking_network_v2.cluster_network.id - - dynamic "fixed_ip" { - for_each = var.ipv4_enabled ? [1] : [] - content { - subnet_id = openstack_networking_subnet_v2.cluster_subnet[0].id - } - } - - dynamic "fixed_ip" { - for_each = var.ipv6_enabled ? [1] : [] - content { - subnet_id = openstack_networking_subnet_v2.cluster_v6_subnet[0].id - } - } - - port_security_enabled = false -} - -resource "openstack_compute_servergroup_v2" "server_group" { - for_each = toset(distinct( - [for k, v in local.worker_nodes : - v.anti_affinity_group if v.anti_affinity_group != null] - )) - name = each.key - policies = ["anti-affinity"] -} - -data "openstack_compute_flavor_v2" "worker" { - for_each = local.worker_nodes - name = each.value.flavor -} - -data "openstack_images_image_v2" "worker" { - for_each = local.worker_nodes - name = each.value.image - most_recent = true -} - -resource "openstack_blockstorage_volume_v3" "worker-volume" { - for_each = var.create_root_disk_on_volume == true ? local.worker_nodes : {} - name = each.value.volume_name - size = (data.openstack_compute_flavor_v2.worker[each.key].disk > 0) ? data.openstack_compute_flavor_v2.worker[each.key].disk : each.value.root_disk_size - image_id = data.openstack_images_image_v2.worker[each.key].id - volume_type = each.value.root_disk_volume_type - availability_zone = each.value.az - - timeouts { - create = var.timeout_time - delete = var.timeout_time - } - - lifecycle { - ignore_changes = [image_id] - } -} - -resource "openstack_compute_instance_v2" "worker" { - for_each = local.worker_nodes - name = each.key - - availability_zone = each.value.az - flavor_id = data.openstack_compute_flavor_v2.worker[each.key].id - image_id = var.create_root_disk_on_volume == false ? data.openstack_images_image_v2.worker[each.key].id : null - key_pair = var.keypair - config_drive = true - - dynamic scheduler_hints { - # Abusing 'for_each' as a conditional - # It's not working as a loop. The outer `each.key` is "passed" into the inner `for_each` - for_each = each.value.anti_affinity_group != null ? [each.key] : [] - content { - group = openstack_compute_servergroup_v2.server_group[each.value.anti_affinity_group].id - } - } - - dynamic block_device { - # Abusing 'for_each' as a conditional - # It's not working as a loop. The outer `each.key` is "passed" into the inner `for_each` - for_each = var.create_root_disk_on_volume == true ? [each.key] : [] - content { - uuid = openstack_blockstorage_volume_v3.worker-volume[each.key].id - source_type = "volume" - boot_index = 0 - destination_type = "volume" - delete_on_termination = true - } - } - - network { - port = openstack_networking_port_v2.worker[each.key].id - } - - # Ignoring 'scheduler_hints' here for existing VMs because otherwise tf would destroy and recreate them. - # The initial distribution for existing clusters must therefore be enforced manually. - lifecycle { - ignore_changes = [key_pair, image_id, config_drive, scheduler_hints] - } -} diff --git a/terraform/50-object-storage.tf b/terraform/50-object-storage.tf deleted file mode 100644 index ec1915fd2da024ec9401f22a7af0e5b14377b2af..0000000000000000000000000000000000000000 --- a/terraform/50-object-storage.tf +++ /dev/null @@ -1,5 +0,0 @@ -resource "openstack_objectstorage_container_v1" "thanos_data" { - name = "${var.cluster_name}-monitoring-thanos-data" - count = var.monitoring_manage_thanos_bucket ? 1 : 0 - force_destroy = var.thanos_delete_container -} diff --git a/terraform/99-outputs.tf b/terraform/99-outputs.tf deleted file mode 100644 index abc32b768c9e9588586dda5cac5227a0a41c0f70..0000000000000000000000000000000000000000 --- a/terraform/99-outputs.tf +++ /dev/null @@ -1,39 +0,0 @@ -resource "local_file" "inventory_yaook-k8s" { - content = templatefile("${path.module}/templates/inventory.tpl", { - masters = openstack_compute_instance_v2.master, - master_ports = openstack_networking_port_v2.master, - gateways = openstack_compute_instance_v2.gateway, - gateway_ports = openstack_networking_port_v2.gateway, - gateway_fips = openstack_networking_floatingip_v2.gateway, - workers = openstack_compute_instance_v2.worker, - worker_ports = openstack_networking_port_v2.worker, - ipv6_enabled = var.ipv6_enabled, - ipv4_enabled = var.ipv4_enabled, - }) - filename = "../../state/terraform/rendered/hosts" - file_permission = 0640 -} - -resource "local_file" "trampoline_gateways" { - content = data.template_file.trampoline_gateways.rendered - filename = "../../state/terraform/rendered/terraform_networking-trampoline.yaml" - file_permission = 0640 -} - -resource "local_file" "final_networking" { - content = templatefile("${path.module}/templates/final_networking.tpl", { - subnet_id = try(openstack_networking_subnet_v2.cluster_subnet[0].id, null), - subnet_v6_id = try(openstack_networking_subnet_v2.cluster_v6_subnet[0].id, null), - floating_ip_network_id = data.openstack_networking_network_v2.public_network.id, - }) - filename = "../../state/terraform/rendered/terraform_networking.yaml" - file_permission = 0640 -} - -# Please note that if gitlab_backend is set to true in the config -# it will override this local backend configuration -terraform { - backend "local" { - path = "../../state/terraform/terraform.tfstate" - } -} diff --git a/terraform/README.md b/terraform/README.md deleted file mode 100644 index 773250f83d9a8f5e63dcdd4d17ed368b8aa9438c..0000000000000000000000000000000000000000 --- a/terraform/README.md +++ /dev/null @@ -1,17 +0,0 @@ -## Bootstrap a cluster - -1. Install terraform -2. Load your openstack credentials into the shell -3. ``terraform init`` -4. ``terraform apply -var keypair=your-openstack-keypair -var subnet_cidr=172.30.154.0/24" - -(You can pick a different subnet, but this’ll do.) - -## Configure nodes - -To execute an ansible stage, make sure you use the terraform inventories. To -run stage 2, you would, *for example*, call -``ansible-playbook -i inventories/terraform/02_trampoline/openstack.yaml 02_trampoline.yaml``. - -1. Set up wireguard (see /docs/admin/wg.md), including running Ansible stage 02 -2. ``ansible-playbook -i inventories/terraform/03_final/openstack.yaml 03_final.yaml --diff -f10`` diff --git a/terraform/templates/final_networking.tpl b/terraform/templates/final_networking.tpl deleted file mode 100644 index 35f0be5555016542ec28e865a6d5148a1f1b07d8..0000000000000000000000000000000000000000 --- a/terraform/templates/final_networking.tpl +++ /dev/null @@ -1,4 +0,0 @@ -openstack_lbaas_subnet_id: ${jsonencode(subnet_id)} -openstack_lbaas_floating_ip_network_id: ${jsonencode(floating_ip_network_id)} -ch_k8s_lbaas_subnet_id: ${jsonencode(subnet_id)} -ch_k8s_lbaas_floating_ip_network_id: ${jsonencode(floating_ip_network_id)} diff --git a/terraform/templates/inventory.tpl b/terraform/templates/inventory.tpl deleted file mode 100644 index 46c1418e045c2844d7c9af313799bd693a10a382..0000000000000000000000000000000000000000 --- a/terraform/templates/inventory.tpl +++ /dev/null @@ -1,27 +0,0 @@ -[all:vars] -ansible_python_interpreter=/usr/bin/python3 - -[orchestrator] -localhost ansible_connection=local ansible_python_interpreter="{{ ansible_playbook_python }}" - -[frontend:children] -gateways - -[k8s_nodes:children] -masters -workers - -[gateways] -%{ for index, instance in gateways ~} -${instance.name} ansible_host=${gateway_fips[index].address} port_id=${gateway_ports[index].id} local_ipv4_address=${gateway_ports[index].all_fixed_ips[0]} %{if ipv6_enabled }${try("local_ipv6_address=${gateway_ports[index].all_fixed_ips[1]}", "")}%{ endif } -%{ endfor } - -[masters] -%{ for index, instance in masters ~} -${instance.name} ansible_host=${master_ports[index].all_fixed_ips[0]} local_ipv4_address=${master_ports[index].all_fixed_ips[0]} %{if ipv6_enabled }${try("local_ipv6_address=${master_ports[index].all_fixed_ips[1]}", "")}%{ endif } -%{ endfor } - -[workers] -%{ for index, instance in workers ~} -${instance.name} ansible_host=${worker_ports[index].all_fixed_ips[0]} local_ipv4_address=${worker_ports[index].all_fixed_ips[0]} %{if ipv6_enabled }${try("local_ipv6_address=${worker_ports[index].all_fixed_ips[1]}", "")}%{ endif } -%{ endfor } diff --git a/terraform/templates/object_storage.tpl b/terraform/templates/object_storage.tpl deleted file mode 100644 index 1369dbf91ee6ded0e12bbc69f27f13eb15be2794..0000000000000000000000000000000000000000 --- a/terraform/templates/object_storage.tpl +++ /dev/null @@ -1 +0,0 @@ -monitoring_thanos_objectstorage_container_name: ${monitoring_thanos_objectstorage_container_name} diff --git a/terraform/templates/trampoline_gateways.tpl b/terraform/templates/trampoline_gateways.tpl deleted file mode 100644 index 9c74d3a92554f54a14945acc3e6e820a8961cfb5..0000000000000000000000000000000000000000 --- a/terraform/templates/trampoline_gateways.tpl +++ /dev/null @@ -1,3 +0,0 @@ -networking_floating_ip : ${networking_floating_ip} -networking_fixed_ip : ${networking_fixed_ip} -networking_fixed_ip_v6: ${networking_fixed_ip_v6} diff --git a/terraform/versions.tf b/terraform/versions.tf deleted file mode 100644 index b60e52955aa70daf2a67af99a763f38062c65e0b..0000000000000000000000000000000000000000 --- a/terraform/versions.tf +++ /dev/null @@ -1,17 +0,0 @@ -terraform { - required_providers { - local = { - source = "hashicorp/local" - version = ">= 2.4.0" - } - openstack = { - source = "terraform-provider-openstack/openstack" - version = "~> 3.3.0" - } - template = { - source = "hashicorp/template" - version = ">= 2.2.0" - } - } - required_version = ">= 0.14" -} diff --git a/tools/vault/import.sh b/tools/vault/import.sh index 3eaf798ec1795ea7ea6088f651406ef681e71c9b..e120a4718a8868df7ad52ca67c537ce258a19216 100755 --- a/tools/vault/import.sh +++ b/tools/vault/import.sh @@ -26,7 +26,7 @@ if [ "$#" -ne "$arg_num" ]; then fi # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" vault cluster="$(get_clustername)" mode="$1" @@ -55,7 +55,7 @@ scriptdir="$(dirname "$0")" inventory_etc=etc flag_file="$inventory_etc/migrated-to-vault" -wg_usage="$(yq '.enabled // true' inventory/yaook-k8s/group_vars/gateways/wireguard.yaml)" +wg_usage="$(yq '.wg_usage' "${vars_file}")" if [ ! -d 'etc' ]; then echo "$0: ./etc does not exist. are you running this from the right place?" >&2 diff --git a/tools/vault/k8s-login.sh b/tools/vault/k8s-login.sh index 57844d90fd6247eac075dca31b898b0c386b3248..70098c84ff91737dcb85b5737940a9d47d4ba022 100755 --- a/tools/vault/k8s-login.sh +++ b/tools/vault/k8s-login.sh @@ -2,6 +2,9 @@ set -euo pipefail actions_dir="$(realpath "$(dirname "$0")")/../../actions" +# Ensure that the latest config is deployed to the inventory +"$actions_dir/update-inventory.sh" vault + # shellcheck source=tools/vault/lib.sh . "$(dirname "$0")/lib.sh" @@ -32,9 +35,6 @@ if [ "$#" -ne "$arg_num" ]; then exit 2 fi -# Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" - cluster="$(get_clustername)" kubernetes_server="$1" username="vault:$(vault token lookup -format=json | jq -r .data.path)" diff --git a/tools/vault/lib.sh b/tools/vault/lib.sh index ec7e4e0ab36ee9afa0b6c312915c89818d73d5e4..ec22f4543ce4a797b4f8b2721fbe1daef1059863 100644 --- a/tools/vault/lib.sh +++ b/tools/vault/lib.sh @@ -2,17 +2,13 @@ set -euo pipefail cluster_repository="$(realpath ".")" -group_vars_dir="${cluster_repository}/inventory/yaook-k8s/group_vars" +vars_file="${cluster_repository}/inventory/vault/main.yaml" common_path_prefix="${YAOOK_K8S_VAULT_PATH_PREFIX:-yaook}" common_policy_prefix="${YAOOK_K8S_VAULT_POLICY_PREFIX:-yaook}" nodes_approle_name="${YAOOK_K8S_VAULT_NODES_APPROLE_NAME:-${common_path_prefix}/nodes}" nodes_approle_path="auth/$nodes_approle_name" -k8s_controller_manager_enable_signing_requests="$( - yq '.k8s_controller_manager_enable_signing_requests - | if (.|type)=="boolean" then . else error("unset-or-invalid") end' \ - "$group_vars_dir/all/kubernetes.yaml" 2>/dev/null -)" || unset k8s_controller_manager_enable_signing_requests # unset when unset, invalid or file missing +k8s_controller_manager_enable_signing_requests="$(yq '.k8s_controller_manager_enable_signing_requests' "$vars_file" 2>/dev/null)" if [ -n "${cluster:-}" ]; then cluster_path="$common_path_prefix/$cluster" @@ -67,7 +63,7 @@ function require_k8s_cluster_ca_backup_destruction { } function get_clustername() { - yq --raw-output '.vault_cluster_name // error("unset")' "${group_vars_dir}/all/vault-backend.yaml" + yq --raw-output '.vault_cluster_name' "${vars_file}" } function init_cluster_secrets_engines() { @@ -467,9 +463,9 @@ function import_ipsec_eap_psk() { } function import_thanos_config() { - thanos_enabled="$(yq '.monitoring_use_thanos' "${group_vars_dir}/all/prometheus.yaml")" - manage_thanos_bucket="$(yq '.monitoring_manage_thanos_bucket' "${group_vars_dir}/all/prometheus.yaml")" - thanos_config_file="$(yq -r '.monitoring_thanos_objectstorage_config_file' "${group_vars_dir}/all/prometheus.yaml")" + thanos_enabled="$(yq '.thanos_enabled' "${vars_file}")" + manage_thanos_bucket="$(yq '.manage_thanos_bucket' "${vars_file}")" + thanos_config_file="$(yq -r '.thanos_config_file' "${vars_file}")" if ! "$thanos_enabled"; then echo "Thanos is disabled." @@ -514,8 +510,8 @@ function import_thanos_config() { } function import_vault_backup_s3_config() { - vault_backup_s3_enabled="$(yq '.yaook_vault_enable_backups' "${group_vars_dir}/all/vault-svc.yaml")" - vault_backup_s3_config_file="$(yq -r '.yaook_vault_s3_config_file' "${group_vars_dir}/all/vault-svc.yaml")" + vault_backup_s3_enabled="$(yq '.vault_backup_s3_enabled' "${vars_file}")" + vault_backup_s3_config_file="$(yq -r '.vault_backup_s3_config_file' "${vars_file}")" if ! "$vault_backup_s3_enabled"; then echo "Vault S3 backup is disabled." diff --git a/tools/vault/load-signed-intermediates.sh b/tools/vault/load-signed-intermediates.sh index 5203eaf4b1b32a89a6a3274758d8f39bada706b8..a355d07fbbfd0403bc393114ac4f337e3aa7f1ff 100755 --- a/tools/vault/load-signed-intermediates.sh +++ b/tools/vault/load-signed-intermediates.sh @@ -13,7 +13,7 @@ if [ "$#" -ne "$arg_num" ]; then fi # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" vault cluster="$(get_clustername)" # reload the lib to update the vars after initializing the clustername diff --git a/tools/vault/mkcluster-intermediate.sh b/tools/vault/mkcluster-intermediate.sh index 067666e8e6b78369d34d7894c5698971a9f3b82f..a8783cf8ff9b9059543755624819617adc549eb4 100755 --- a/tools/vault/mkcluster-intermediate.sh +++ b/tools/vault/mkcluster-intermediate.sh @@ -13,7 +13,7 @@ if [ "$#" -ne "$arg_num" ]; then fi # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" vault cluster="$(get_clustername)" # reload the lib to update the vars after initializing the clustername diff --git a/tools/vault/mkcluster-root.sh b/tools/vault/mkcluster-root.sh index 6b5036264ed849c61f96ab3c40f2b2e371465c37..dbcaa10c69e676def3453318a1e2b6b045e0a214 100755 --- a/tools/vault/mkcluster-root.sh +++ b/tools/vault/mkcluster-root.sh @@ -13,7 +13,7 @@ if [ "$#" -ne "$arg_num" ]; then fi # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" vault cluster="$(get_clustername)" # reload the lib to update the vars after initializing the clustername diff --git a/tools/vault/rmcluster.sh b/tools/vault/rmcluster.sh index 24cd8588927ca4cbdd25453f5f8b301afa7e60aa..06410a8ce6060ac99aae134a14b0fc95949e153a 100755 --- a/tools/vault/rmcluster.sh +++ b/tools/vault/rmcluster.sh @@ -13,7 +13,7 @@ if [ "$#" -ne "$arg_num" ]; then fi # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" vault cluster="$(get_clustername)" # reload the lib to update the vars after initializing the clustername diff --git a/tools/vault/rotate-root-ca-intermediate.sh b/tools/vault/rotate-root-ca-intermediate.sh index f77737e201d90fbf427a24e611efa63501a3b2df..d517d3217be6f5d68669fb123279cbfa4a851f84 100755 --- a/tools/vault/rotate-root-ca-intermediate.sh +++ b/tools/vault/rotate-root-ca-intermediate.sh @@ -18,7 +18,7 @@ if [ "$#" -ne "$arg_num" ]; then fi # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" vault cluster="$(get_clustername)" # reload the lib to update the vars after initializing the clustername diff --git a/tools/vault/rotate-root-ca-root.sh b/tools/vault/rotate-root-ca-root.sh index 3e0419eaa390b0f662c32a9e8bbb38487365ceaf..0cdb1a822186726d224699059d81fb0196ae3ae8 100755 --- a/tools/vault/rotate-root-ca-root.sh +++ b/tools/vault/rotate-root-ca-root.sh @@ -19,7 +19,7 @@ if [ "$#" -ne "$arg_num" ]; then fi # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" vault cluster="$(get_clustername)" # reload the lib to update the vars after initializing the clustername diff --git a/tools/vault/update.sh b/tools/vault/update.sh index b7283b30290e7afd08eaea59513d32fb1f3ebf79..508eba2b82a64381f170a7ba3fa1aac1f04f40c9 100755 --- a/tools/vault/update.sh +++ b/tools/vault/update.sh @@ -13,7 +13,7 @@ if [ "$#" -ne "$arg_num" ]; then fi # Ensure that the latest config is deployed to the inventory -"$actions_dir/update-inventory.sh" +"$actions_dir/update-inventory.sh" vault cluster="$(get_clustername)" # reload the lib to update the vars after initializing the clustername