diff --git a/app/serializers/deployment_entity.rb b/app/serializers/deployment_entity.rb index dc7c4654208afa69361022bc7fcb8b552550d609..a37011d01006a2fd071f375ba1583c0984b1e24c 100644 --- a/app/serializers/deployment_entity.rb +++ b/app/serializers/deployment_entity.rb @@ -17,6 +17,7 @@ class DeploymentEntity < Grape::Entity end end + expose :status expose :created_at expose :deployed_at expose :tag diff --git a/config/feature_flags/development/canary_ingress_weight_control.yml b/config/feature_flags/development/canary_ingress_weight_control.yml new file mode 100644 index 0000000000000000000000000000000000000000..681ffc98cb5d7e932cd3072694b3487189b10535 --- /dev/null +++ b/config/feature_flags/development/canary_ingress_weight_control.yml @@ -0,0 +1,7 @@ +--- +name: canary_ingress_weight_control +introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/43816 +rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/260295 +type: development +group: group::progressive delivery +default_enabled: false diff --git a/ee/app/models/ee/clusters/platforms/kubernetes.rb b/ee/app/models/ee/clusters/platforms/kubernetes.rb index 632fed925c903e26769417f5fc8114459e1e6a2e..b4d54d0a452e0dc47714cc417e417030a3a0ee9a 100644 --- a/ee/app/models/ee/clusters/platforms/kubernetes.rb +++ b/ee/app/models/ee/clusters/platforms/kubernetes.rb @@ -14,8 +14,15 @@ def calculate_reactive_cache_for(environment) if result deployments = read_deployments(environment.deployment_namespace) + ingresses = if ::Feature.enabled?(:canary_ingress_weight_control, environment.project) + read_ingresses(environment.deployment_namespace) + else + [] + end + # extract_relevant_deployment_data avoids uploading all the deployment info into ReactiveCaching result[:deployments] = extract_relevant_deployment_data(deployments) + result[:ingresses] = extract_relevant_ingress_data(ingresses) end result @@ -26,8 +33,9 @@ def rollout_status(environment, data) deployments = filter_by_project_environment(data[:deployments], project.full_path_slug, environment.slug) pods = filter_by_project_environment(data[:pods], project.full_path_slug, environment.slug) + ingresses = data[:ingresses].presence || [] - ::Gitlab::Kubernetes::RolloutStatus.from_deployments(*deployments, pods_attrs: pods) + ::Gitlab::Kubernetes::RolloutStatus.from_deployments(*deployments, pods_attrs: pods, ingresses: ingresses) end private @@ -38,6 +46,12 @@ def read_deployments(namespace) [] end + def read_ingresses(namespace) + kubeclient.get_ingresses(namespace: namespace).as_json + rescue Kubeclient::ResourceNotFoundError + [] + end + def extract_relevant_deployment_data(deployments) deployments.map do |deployment| { @@ -47,6 +61,14 @@ def extract_relevant_deployment_data(deployments) } end end + + def extract_relevant_ingress_data(ingresses) + ingresses.map do |ingress| + { + 'metadata' => ingress.fetch('metadata', {}).slice('name', 'labels', 'annotations') + } + end + end end end end diff --git a/ee/app/serializers/rollout_status_entity.rb b/ee/app/serializers/rollout_status_entity.rb index d2c2e16d66a979b20dfff4e2dc59612561fe738f..9f4c844859bb233ce6769f5f930dda2d40fe7969 100644 --- a/ee/app/serializers/rollout_status_entity.rb +++ b/ee/app/serializers/rollout_status_entity.rb @@ -13,4 +13,6 @@ class RolloutStatusEntity < Grape::Entity expose :instances, if: -> (rollout_status, _) { rollout_status.found? } expose :completion, if: -> (rollout_status, _) { rollout_status.found? } expose :complete?, as: :is_completed, if: -> (rollout_status, _) { rollout_status.found? } + expose :canary_ingress, using: RolloutStatuses::IngressEntity, expose_nil: false, + if: -> (rollout_status, _) { rollout_status.found? && rollout_status.canary_ingress_exists? } end diff --git a/ee/app/serializers/rollout_statuses/ingress_entity.rb b/ee/app/serializers/rollout_statuses/ingress_entity.rb new file mode 100644 index 0000000000000000000000000000000000000000..a68d936b86cf47dc997eff0a0a71f4e96e0089cf --- /dev/null +++ b/ee/app/serializers/rollout_statuses/ingress_entity.rb @@ -0,0 +1,7 @@ +# frozen_string_literal: true + +module RolloutStatuses + class IngressEntity < Grape::Entity + expose :canary_weight + end +end diff --git a/ee/lib/gitlab/kubernetes/ingress.rb b/ee/lib/gitlab/kubernetes/ingress.rb new file mode 100644 index 0000000000000000000000000000000000000000..581116bbc6b65db61496e1a7510763539599f1ae --- /dev/null +++ b/ee/lib/gitlab/kubernetes/ingress.rb @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +module Gitlab + module Kubernetes + class Ingress + include Gitlab::Utils::StrongMemoize + + # Canry Ingress Annotations https://kubernetes.github.io/ingress-nginx/user-guide/nginx-configuration/annotations/#canary + ANNOTATION_KEY_CANARY = 'nginx.ingress.kubernetes.io/canary' + ANNOTATION_KEY_CANARY_WEIGHT = 'nginx.ingress.kubernetes.io/canary-weight' + + def initialize(attributes = {}) + @attributes = attributes + end + + def canary? + strong_memoize(:is_canary) do + annotations.any? do |key, value| + key == ANNOTATION_KEY_CANARY && value == 'true' + end + end + end + + def canary_weight + return unless canary? + return unless annotations.key?(ANNOTATION_KEY_CANARY_WEIGHT) + + annotations[ANNOTATION_KEY_CANARY_WEIGHT].to_i + end + + private + + def metadata + @attributes.fetch('metadata', {}) + end + + def annotations + metadata.fetch('annotations', {}) + end + end + end +end diff --git a/ee/lib/gitlab/kubernetes/rollout_status.rb b/ee/lib/gitlab/kubernetes/rollout_status.rb index 6ec001977e3db0040752682b1b3c318f39bc611c..9db132a4da0cef0b8633ce20d6fcacb4ee30d03d 100644 --- a/ee/lib/gitlab/kubernetes/rollout_status.rb +++ b/ee/lib/gitlab/kubernetes/rollout_status.rb @@ -8,7 +8,7 @@ module Kubernetes # other resources. The rollout status sums the Kubernetes deployments # together. class RolloutStatus - attr_reader :deployments, :instances, :completion, :status + attr_reader :deployments, :instances, :completion, :status, :canary_ingress def complete? completion == 100 @@ -26,7 +26,11 @@ def found? @status == :found end - def self.from_deployments(*deployments_attrs, pods_attrs: []) + def canary_ingress_exists? + canary_ingress.present? + end + + def self.from_deployments(*deployments_attrs, pods_attrs: [], ingresses: []) return new([], status: :not_found) if deployments_attrs.empty? deployments = deployments_attrs.map do |attrs| @@ -38,14 +42,16 @@ def self.from_deployments(*deployments_attrs, pods_attrs: []) ::Gitlab::Kubernetes::Pod.new(attrs) end - new(deployments, pods: pods) + ingresses = ingresses.map { |ingress| ::Gitlab::Kubernetes::Ingress.new(ingress) } + + new(deployments, pods: pods, ingresses: ingresses) end def self.loading new([], status: :loading) end - def initialize(deployments, pods: [], status: :found) + def initialize(deployments, pods: [], ingresses: [], status: :found) @status = status @deployments = deployments @@ -55,6 +61,8 @@ def initialize(deployments, pods: [], status: :found) deployments.flat_map(&:instances) end + @canary_ingress = ingresses.find(&:canary?) + @completion = if @instances.empty? 100 diff --git a/ee/spec/lib/gitlab/kubernetes/ingress_spec.rb b/ee/spec/lib/gitlab/kubernetes/ingress_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..12eb562d68bbc78fedbee00cbc688a4c7438eec2 --- /dev/null +++ b/ee/spec/lib/gitlab/kubernetes/ingress_spec.rb @@ -0,0 +1,49 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe Gitlab::Kubernetes::Ingress do + include KubernetesHelpers + + let(:ingress) { described_class.new(params) } + + describe '#canary?' do + subject { ingress.canary? } + + context 'with canary ingress parameters' do + let(:params) { canary_metadata } + + it { is_expected.to be_truthy } + end + + context 'with stable ingress parameters' do + let(:params) { stable_metadata } + + it { is_expected.to be_falsey } + end + end + + describe '#canary_weight' do + subject { ingress.canary_weight } + + context 'with canary ingress parameters' do + let(:params) { canary_metadata } + + it { is_expected.to eq(50) } + end + + context 'with stable ingress parameters' do + let(:params) { stable_metadata } + + it { is_expected.to be_nil } + end + end + + def stable_metadata + kube_ingress(track: :stable) + end + + def canary_metadata + kube_ingress(track: :canary) + end +end diff --git a/ee/spec/lib/gitlab/kubernetes/rollout_status_spec.rb b/ee/spec/lib/gitlab/kubernetes/rollout_status_spec.rb index d5a8f1d3b18c02fda4cbe731665b6dc33d54f166..9a3682b08acb69349b96f45c2a3bb378bbb77170 100644 --- a/ee/spec/lib/gitlab/kubernetes/rollout_status_spec.rb +++ b/ee/spec/lib/gitlab/kubernetes/rollout_status_spec.rb @@ -12,6 +12,8 @@ create_pods(name: "one", count: 3, track: 'stable') + create_pods(name: "two", count: 3, track: "canary") end + let(:ingresses) { [] } + let(:specs_all_finished) do [ kube_deployment(name: 'one'), @@ -26,7 +28,7 @@ ] end - subject(:rollout_status) { described_class.from_deployments(*specs, pods_attrs: pods) } + subject(:rollout_status) { described_class.from_deployments(*specs, pods_attrs: pods, ingresses: ingresses) } shared_examples 'rollout status' do describe '#deployments' do @@ -223,6 +225,24 @@ it { is_expected.to be_not_found } end end + + describe '#canary_ingress_exists?' do + context 'when canary ingress exists' do + let(:ingresses) { [kube_ingress(track: :canary)] } + + it 'returns true' do + expect(rollout_status.canary_ingress_exists?).to eq(true) + end + end + + context 'when canary ingress does not exist' do + let(:ingresses) { [kube_ingress(track: :stable)] } + + it 'returns false' do + expect(rollout_status.canary_ingress_exists?).to eq(false) + end + end + end end context 'deploy_boards_dedupe_instances is disabled' do diff --git a/ee/spec/models/ee/clusters/platforms/kubernetes_spec.rb b/ee/spec/models/ee/clusters/platforms/kubernetes_spec.rb index 291be5013180d3faa48d564bb5ae0ccd38fa27ba..565811423faae8653a0e13c71f2010cb5785d097 100644 --- a/ee/spec/models/ee/clusters/platforms/kubernetes_spec.rb +++ b/ee/spec/models/ee/clusters/platforms/kubernetes_spec.rb @@ -27,11 +27,12 @@ describe '#rollout_status' do let(:deployments) { [] } let(:pods) { [] } + let(:ingresses) { [] } let(:service) { create(:cluster_platform_kubernetes, :configured) } let!(:cluster) { create(:cluster, :project, enabled: true, platform_kubernetes: service) } let(:project) { cluster.project } let(:environment) { build(:environment, project: project, name: "env", slug: "env-000000") } - let(:cache_data) { Hash(deployments: deployments, pods: pods) } + let(:cache_data) { Hash(deployments: deployments, pods: pods, ingresses: ingresses) } subject(:rollout_status) { service.rollout_status(environment, cache_data) } @@ -129,6 +130,15 @@ tooltip: "Not provided (Pending)", track: "stable" }]) end + + context 'with canary ingress' do + let(:ingresses) { [kube_ingress(track: :canary)] } + + it 'has canary ingress' do + expect(rollout_status).to be_canary_ingress_exists + expect(rollout_status.canary_ingress.canary_weight).to eq(50) + end + end end context 'with empty list of deployments' do @@ -304,7 +314,7 @@ let(:cluster) { create(:cluster, :project, platform_kubernetes: service) } let(:service) { create(:cluster_platform_kubernetes, :configured) } let(:namespace) { 'project-namespace' } - let(:environment) { instance_double(Environment, deployment_namespace: namespace) } + let(:environment) { instance_double(Environment, deployment_namespace: namespace, project: cluster.project) } let(:expected_pod_cached_data) do kube_pod.tap { |kp| kp['metadata'].delete('namespace') } end @@ -315,10 +325,11 @@ before do stub_kubeclient_pods(namespace) stub_kubeclient_deployments(namespace) + stub_kubeclient_ingresses(namespace) end shared_examples 'successful deployment request' do - it { is_expected.to include(pods: [expected_pod_cached_data], deployments: [kube_deployment]) } + it { is_expected.to include(pods: [expected_pod_cached_data], deployments: [kube_deployment], ingresses: [kube_ingress]) } end context 'on a project level cluster' do @@ -338,6 +349,16 @@ include_examples 'successful deployment request' end + + context 'when canary_ingress_weight_control feature flag is disabled' do + before do + stub_feature_flags(canary_ingress_weight_control: false) + end + + it 'does not fetch ingress data from kubernetes' do + expect(subject[:ingresses]).to be_empty + end + end end context 'when kubernetes responds with 500s' do @@ -353,9 +374,10 @@ before do stub_kubeclient_pods(namespace) stub_kubeclient_deployments(namespace, status: 404) + stub_kubeclient_ingresses(namespace, status: 404) end - it { is_expected.to include(deployments: []) } + it { is_expected.to include(deployments: [], ingresses: []) } end end end diff --git a/ee/spec/serializers/rollout_status_entity_spec.rb b/ee/spec/serializers/rollout_status_entity_spec.rb index e78c63ebc5e23e4dd06e545007a5df5a615a7518..7ad4b259bcd0aa898eb90dc140699075e15607d6 100644 --- a/ee/spec/serializers/rollout_status_entity_spec.rb +++ b/ee/spec/serializers/rollout_status_entity_spec.rb @@ -25,6 +25,18 @@ it "exposes deployment data" do is_expected.to include(:instances, :completion, :is_completed) end + + it 'does not expose canary ingress if it does not exist' do + is_expected.not_to include(:canary_ingress) + end + + context 'when canary ingress exists' do + let(:rollout_status) { kube_deployment_rollout_status(ingresses: [kube_ingress(track: :canary)]) } + + it 'expose canary ingress' do + is_expected.to include(:canary_ingress) + end + end end context 'when kube deployment is empty' do @@ -35,7 +47,7 @@ end it "does not expose deployment data" do - is_expected.not_to include(:instances, :completion, :is_completed) + is_expected.not_to include(:instances, :completion, :is_completed, :canary_ingress) end end end diff --git a/ee/spec/serializers/rollout_statuses/ingress_entity_spec.rb b/ee/spec/serializers/rollout_statuses/ingress_entity_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..b87b9e5c6c42338e89af72cc101bfba292a658e0 --- /dev/null +++ b/ee/spec/serializers/rollout_statuses/ingress_entity_spec.rb @@ -0,0 +1,19 @@ +# frozen_string_literal: true + +require 'spec_helper' + +RSpec.describe RolloutStatuses::IngressEntity do + include KubernetesHelpers + + let(:canary_ingress) { kube_ingress(track: :canary) } + + let(:entity) do + described_class.new(canary_ingress, request: double) + end + + subject { entity.as_json } + + it 'exposes canary weight' do + is_expected.to include(:canary_weight) + end +end diff --git a/lib/gitlab/kubernetes/kube_client.rb b/lib/gitlab/kubernetes/kube_client.rb index fa68afd39f5ac06db2e1a44a43b6268dc2d6ad75..13cd6dcad3f95adb727318f83eee3fcf56305c33 100644 --- a/lib/gitlab/kubernetes/kube_client.rb +++ b/lib/gitlab/kubernetes/kube_client.rb @@ -167,6 +167,21 @@ def get_deployments(**args) end end + # Ingresses resource is currently on the apis/extensions api group + # until Kubernetes 1.21. Kubernetest 1.22+ has ingresses resources in + # the networking.k8s.io/v1 api group. + # + # As we still support Kubernetes 1.12+, we will need to support both. + def get_ingresses(**args) + extensions_client.discover unless extensions_client.discovered + + if extensions_client.respond_to?(:get_ingresses) + extensions_client.get_ingresses(**args) + else + networking_client.get_ingresses(**args) + end + end + def create_or_update_cluster_role_binding(resource) update_cluster_role_binding(resource) end diff --git a/spec/lib/gitlab/kubernetes/kube_client_spec.rb b/spec/lib/gitlab/kubernetes/kube_client_spec.rb index b161832c018d7f4e7c8e0bd243e23e37b2290bf6..7b6d143dda95a0817c8b97d5200c46fc683aad30 100644 --- a/spec/lib/gitlab/kubernetes/kube_client_spec.rb +++ b/spec/lib/gitlab/kubernetes/kube_client_spec.rb @@ -347,6 +347,34 @@ def method_call(client, method_name) end end + describe '#get_ingresses' do + let(:extensions_client) { client.extensions_client } + let(:networking_client) { client.networking_client } + + include_examples 'redirection not allowed', 'get_ingresses' + include_examples 'dns rebinding not allowed', 'get_ingresses' + + it 'delegates to the extensions client' do + expect(extensions_client).to receive(:get_ingresses) + + client.get_ingresses + end + + context 'extensions does not have deployments for Kubernetes 1.22+ clusters' do + before do + WebMock + .stub_request(:get, api_url + '/apis/extensions/v1beta1') + .to_return(kube_response(kube_1_22_extensions_v1beta1_discovery_body)) + end + + it 'delegates to the apps client' do + expect(networking_client).to receive(:get_ingresses) + + client.get_ingresses + end + end + end + describe 'istio API group' do let(:istio_client) { client.istio_client } diff --git a/spec/models/clusters/platforms/kubernetes_spec.rb b/spec/models/clusters/platforms/kubernetes_spec.rb index c6a2b67a0084b20cb645d4c68b3532008931effb..e877ba2ac966648d755d742dbdba4df04e2c5d96 100644 --- a/spec/models/clusters/platforms/kubernetes_spec.rb +++ b/spec/models/clusters/platforms/kubernetes_spec.rb @@ -412,7 +412,7 @@ end let(:namespace) { "project-namespace" } - let(:environment) { instance_double(Environment, deployment_namespace: namespace) } + let(:environment) { instance_double(Environment, deployment_namespace: namespace, project: service.cluster.project) } subject { service.calculate_reactive_cache_for(environment) } @@ -428,6 +428,7 @@ before do stub_kubeclient_pods(namespace) stub_kubeclient_deployments(namespace) + stub_kubeclient_ingresses(namespace) end it { is_expected.to include(pods: [expected_pod_cached_data]) } @@ -437,6 +438,7 @@ before do stub_kubeclient_pods(namespace, status: 500) stub_kubeclient_deployments(namespace, status: 500) + stub_kubeclient_ingresses(namespace, status: 500) end it { expect { subject }.to raise_error(Kubeclient::HttpError) } @@ -446,6 +448,7 @@ before do stub_kubeclient_pods(namespace, status: 404) stub_kubeclient_deployments(namespace, status: 404) + stub_kubeclient_ingresses(namespace, status: 404) end it { is_expected.to include(pods: []) } diff --git a/spec/serializers/deployment_entity_spec.rb b/spec/serializers/deployment_entity_spec.rb index 27673b905d35ed32712b938e0a8b5c773bdd46ac..588675f5232a9986c93b13ae467e18bfe176348a 100644 --- a/spec/serializers/deployment_entity_spec.rb +++ b/spec/serializers/deployment_entity_spec.rb @@ -30,6 +30,10 @@ expect(subject[:ref][:name]).to eq 'master' end + it 'exposes status' do + expect(subject).to include(:status) + end + it 'exposes creation date' do expect(subject).to include(:created_at) end diff --git a/spec/support/helpers/kubernetes_helpers.rb b/spec/support/helpers/kubernetes_helpers.rb index 90ddab89943c6773b75d5ad51cc7a462edb4c960..113bb31e4be74b47a3addf8e1673c123a5584652 100644 --- a/spec/support/helpers/kubernetes_helpers.rb +++ b/spec/support/helpers/kubernetes_helpers.rb @@ -33,6 +33,10 @@ def kube_deployments_response kube_response(kube_deployments_body) end + def kube_ingresses_response + kube_response(kube_ingresses_body) + end + def stub_kubeclient_discover_base(api_url) WebMock.stub_request(:get, api_url + '/api/v1').to_return(kube_response(kube_v1_discovery_body)) WebMock @@ -63,6 +67,9 @@ def stub_kubeclient_discover(api_url) WebMock .stub_request(:get, api_url + '/apis/serving.knative.dev/v1alpha1') .to_return(kube_response(kube_v1alpha1_serving_knative_discovery_body)) + WebMock + .stub_request(:get, api_url + '/apis/networking.k8s.io/v1') + .to_return(kube_response(kube_v1_networking_discovery_body)) end def stub_kubeclient_discover_knative_not_found(api_url) @@ -148,6 +155,14 @@ def stub_kubeclient_deployments(namespace, status: nil) WebMock.stub_request(:get, deployments_url).to_return(response || kube_deployments_response) end + def stub_kubeclient_ingresses(namespace, status: nil) + stub_kubeclient_discover(service.api_url) + ingresses_url = service.api_url + "/apis/extensions/v1beta1/namespaces/#{namespace}/ingresses" + response = { status: status } if status + + WebMock.stub_request(:get, ingresses_url).to_return(response || kube_ingresses_response) + end + def stub_kubeclient_knative_services(options = {}) namespace_path = options[:namespace].present? ? "namespaces/#{options[:namespace]}/" : "" @@ -304,6 +319,14 @@ def kube_1_16_extensions_v1beta1_discovery_body } end + # From Kubernetes 1.22+ Ingresses are no longer served from apis/extensions + def kube_1_22_extensions_v1beta1_discovery_body + { + "kind" => "APIResourceList", + "resources" => [] + } + end + def kube_knative_discovery_body { "kind" => "APIResourceList", @@ -416,6 +439,17 @@ def kube_istio_discovery_body } end + def kube_v1_networking_discovery_body + { + "kind" => "APIResourceList", + "apiVersion" => "v1", + "groupVersion" => "networking.k8s.io/v1", + "resources" => [ + { "name" => "ingresses", "namespaced" => true, "kind" => "Ingress" } + ] + } + end + def kube_istio_gateway_body(name, namespace) { "apiVersion" => "networking.istio.io/v1alpha3", @@ -507,6 +541,13 @@ def kube_deployments_body } end + def kube_ingresses_body + { + "kind" => "List", + "items" => [kube_ingress] + } + end + def kube_knative_pods_body(name, namespace) { "kind" => "PodList", @@ -548,6 +589,38 @@ def kube_pod(name: "kube-pod", container_name: "container-0", environment_slug: } end + def kube_ingress(track: :stable) + additional_annotations = + if track == :canary + { + "nginx.ingress.kubernetes.io/canary" => "true", + "nginx.ingress.kubernetes.io/canary-by-header" => "canary", + "nginx.ingress.kubernetes.io/canary-weight" => "50" + } + else + {} + end + + { + "metadata" => { + "name" => "production-auto-deploy", + "labels" => { + "app" => "production", + "app.kubernetes.io/managed-by" => "Helm", + "chart" => "auto-deploy-app-2.0.0-beta.2", + "heritage" => "Helm", + "release" => "production" + }, + "annotations" => { + "kubernetes.io/ingress.class" => "nginx", + "kubernetes.io/tls-acme" => "true", + "meta.helm.sh/release-name" => "production", + "meta.helm.sh/release-namespace" => "awesome-app-1-production" + }.merge(additional_annotations) + } + } + end + # This is a partial response, it will have many more elements in reality but # these are the ones we care about at the moment def kube_node @@ -862,8 +935,8 @@ def kube_terminals(service, pod) end end - def kube_deployment_rollout_status - ::Gitlab::Kubernetes::RolloutStatus.from_deployments(kube_deployment) + def kube_deployment_rollout_status(ingresses: []) + ::Gitlab::Kubernetes::RolloutStatus.from_deployments(kube_deployment, ingresses: ingresses) end def empty_deployment_rollout_status