From 404da50ba64d3912c760c0be80af75c31f46b3cb Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Sat, 15 Mar 2025 16:56:59 +0000 Subject: [PATCH 1/4] add L7 log collection and forwarding to envoy gateway --- pkg/render/gateway_api.go | 129 +++++++++++++++++++++++++++++++++++++- 1 file changed, 127 insertions(+), 2 deletions(-) diff --git a/pkg/render/gateway_api.go b/pkg/render/gateway_api.go index e5f0db7456..feb7b374c8 100644 --- a/pkg/render/gateway_api.go +++ b/pkg/render/gateway_api.go @@ -27,6 +27,7 @@ import ( rcomp "github.com/tigera/operator/pkg/render/common/components" rmeta "github.com/tigera/operator/pkg/render/common/meta" "github.com/tigera/operator/pkg/render/common/secret" + "github.com/tigera/operator/pkg/render/common/securitycontext" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" @@ -35,6 +36,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" gapi "sigs.k8s.io/gateway-api/apis/v1" + gwapiv1 "sigs.k8s.io/gateway-api/apis/v1" "sigs.k8s.io/yaml" // gopkg.in/yaml.v2 didn't parse all the fields but this package did ) @@ -78,6 +80,8 @@ const ( EnvoyGatewayConfigKey = "envoy-gateway.yaml" EnvoyGatewayDeploymentContainerName = "envoy-gateway" EnvoyGatewayJobContainerName = "envoy-gateway-certgen" + EnvoyGatewayNamespace = "tigera-gateway" + FelixSync = "felix-sync" ) func GatewayAPIResourcesGetter() func() *gatewayAPIResources { @@ -336,6 +340,7 @@ type gatewayAPIImplementationComponent struct { envoyGatewayImage string envoyProxyImage string envoyRatelimitImage string + logCollectorImage string } func GatewayAPIImplementationComponent(cfg *GatewayAPIImplementationConfig) Component { @@ -494,6 +499,9 @@ func (pr *gatewayAPIImplementationComponent) Objects() ([]client.Object, []clien objs = append(objs, certgenJob) + // Provision the log collector deployment and service. + objs = append(objs, pr.logCollectorDeploymentAndService()...) + // Provision a GatewayClass that references the EnvoyProxy config and the controllerName // that the gateway controller expects. objs = append(objs, pr.gatewayClass(envoyGatewayConfig.Gateway.ControllerName, proxyConfig)) @@ -506,9 +514,10 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig() *envoyapi.EnvoyP TypeMeta: metav1.TypeMeta{Kind: "EnvoyProxy", APIVersion: "gateway.envoyproxy.io/v1alpha1"}, ObjectMeta: metav1.ObjectMeta{ Name: "envoy-proxy-config", - Namespace: "tigera-gateway", + Namespace: EnvoyGatewayNamespace, }, Spec: envoyapi.EnvoyProxySpec{ + Telemetry: pr.proxyTelemetrySettings(), Provider: &envoyapi.EnvoyProxyProvider{ Type: envoyapi.ProviderTypeKubernetes, Kubernetes: &envoyapi.EnvoyProxyKubernetesProvider{ @@ -530,12 +539,128 @@ func (pr *gatewayAPIImplementationComponent) envoyProxyConfig() *envoyapi.EnvoyP return envoyProxy } +func (pr *gatewayAPIImplementationComponent) logCollectorVolumes() []corev1.Volume { + return []corev1.Volume{ + { + Name: FelixSync, + VolumeSource: corev1.VolumeSource{ + CSI: &corev1.CSIVolumeSource{ + Driver: "csi.tigera.io", + }, + }, + }, + } +} + +func (pr *gatewayAPIImplementationComponent) logCollectorEnv() []corev1.EnvVar { + envs := []corev1.EnvVar{ + {Name: "LOG_LEVEL", Value: "Info"}, + {Name: "FELIX_DIAL_TARGET", Value: "/var/run/felix/nodeagent/socket"}, + // only use ALS gRPC endpoint + {Name: "LISTEN_ADDRESS", Value: ":8080"}, + {Name: "LISTEN_NETWORK", Value: "tcp"}, + } + return envs +} + +func (pr *gatewayAPIImplementationComponent) logCollectorVolMounts() []corev1.VolumeMount { + return []corev1.VolumeMount{ + {Name: FelixSync, MountPath: "/var/run/felix"}, + } +} + +func (pr *gatewayAPIImplementationComponent) logCollectorDeploymentAndService() []client.Object { + var replicas int32 = 2 + + return []client.Object{ + // The log collector deployment. + &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-l7-log-collector", + Namespace: EnvoyGatewayNamespace, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: &replicas, + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "tigera-l7-log-collector"}, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"app": "tigera-l7-log-collector"}}, + Spec: corev1.PodSpec{ + Volumes: pr.logCollectorVolumes(), + Containers: []corev1.Container{ + { + Name: "tigera-l7-log-collector", + Image: pr.logCollectorImage, + Env: pr.logCollectorEnv(), + SecurityContext: securitycontext.NewRootContext(false), + VolumeMounts: pr.logCollectorVolMounts(), + }, + }, + }, + }, + }, + }, + // The log collector service. + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tigera-l7-log-collector-service", + Namespace: EnvoyGatewayNamespace, + }, + Spec: corev1.ServiceSpec{ + Selector: map[string]string{"app": "tigera-l7-log-collector"}, + Ports: []corev1.ServicePort{ + { + Name: "https", + Port: 8080, + }, + }, + }, + }, + } +} + +func (pr *gatewayAPIImplementationComponent) proxyTelemetrySettings() *envoyapi.ProxyTelemetry { + // The log collector service that the envoy proxy will send logs to. + logCollectSvcName := gwapiv1.ObjectName("tigera-l7-log-collector-service") + logCollectSvcNamespace := gwapiv1.Namespace(EnvoyGatewayNamespace) + logCollectSvcPort := gwapiv1.PortNumber(8080) + + return &envoyapi.ProxyTelemetry{ + AccessLog: &envoyapi.ProxyAccessLog{ + Settings: []envoyapi.ProxyAccessLogSetting{ + { + Sinks: []envoyapi.ProxyAccessLogSink{ + { + Type: envoyapi.ProxyAccessLogSinkTypeALS, + ALS: &envoyapi.ALSEnvoyProxyAccessLog{ + Type: envoyapi.ALSEnvoyProxyAccessLogTypeHTTP, + BackendCluster: envoyapi.BackendCluster{ + BackendRefs: []envoyapi.BackendRef{ + { + BackendObjectReference: gwapiv1.BackendObjectReference{ + Name: logCollectSvcName, + Namespace: &logCollectSvcNamespace, + Port: &logCollectSvcPort, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } +} + func (pr *gatewayAPIImplementationComponent) gatewayClass(controllerName string, proxyConfig *envoyapi.EnvoyProxy) *gapi.GatewayClass { return &gapi.GatewayClass{ TypeMeta: metav1.TypeMeta{Kind: "GatewayClass", APIVersion: "gateway.networking.k8s.io/v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "tigera-gateway-class", - Namespace: "tigera-gateway", + Namespace: EnvoyGatewayNamespace, }, Spec: gapi.GatewayClassSpec{ ControllerName: gapi.GatewayController(controllerName), From a93091b637fb11dbac2fe470cd7bf32f7110a1da Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Thu, 3 Apr 2025 22:59:16 +0100 Subject: [PATCH 2/4] monitor: policy on envoygateway port 19001, add podmonitors --- pkg/render/monitor/monitor.go | 79 +++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/pkg/render/monitor/monitor.go b/pkg/render/monitor/monitor.go index 0eb3892c46..699cdbfa23 100644 --- a/pkg/render/monitor/monitor.go +++ b/pkg/render/monitor/monitor.go @@ -90,6 +90,10 @@ const ( bearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" KubeControllerMetrics = "calico-kube-controllers-metrics" + + TigeraGatewayPodMonitor = "tigera-gateway-pod-monitor" + TigeraGatewayComponentsMetricsPort = 19001 + TigeraGatewayNamespace = "tigera-gateway" ) var alertManagerSelector = fmt.Sprintf( @@ -223,6 +227,8 @@ func (mc *monitorComponent) Objects() ([]client.Object, []client.Object) { mc.serviceMonitorFluentd(), mc.serviceMonitorQueryServer(), mc.serviceMonitorCalicoKubeControllers(), + mc.podMonitorTigeraGatewayAPIController(), + mc.podMonitorTigeraGatewayGateways(), ) if mc.cfg.KeyValidatorConfig != nil { @@ -817,6 +823,71 @@ func (mc *monitorComponent) prometheusRule() *monitoringv1.PrometheusRule { } } +func (mc *monitorComponent) podMonitorTigeraGatewayAPIController() *monitoringv1.PodMonitor { + return &monitoringv1.PodMonitor{ + TypeMeta: metav1.TypeMeta{Kind: monitoringv1.PodMonitorsKind, APIVersion: MonitoringAPIVersion}, + ObjectMeta: metav1.ObjectMeta{ + Name: TigeraGatewayPodMonitor, + Namespace: common.TigeraPrometheusNamespace, + Labels: map[string]string{ + "team": "network-operators", + }, + }, + Spec: monitoringv1.PodMonitorSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "k8s-app": "gateway-api-controller", + }, + }, + NamespaceSelector: monitoringv1.NamespaceSelector{ + MatchNames: []string{TigeraGatewayNamespace}, + }, + PodMetricsEndpoints: []monitoringv1.PodMetricsEndpoint{ + { + Port: "metrics", + ScrapeTimeout: "5s", + Interval: "5s", + Scheme: "http", + HonorLabels: true, + }, + }, + }, + } +} + +func (mc *monitorComponent) podMonitorTigeraGatewayGateways() *monitoringv1.PodMonitor { + return &monitoringv1.PodMonitor{ + TypeMeta: metav1.TypeMeta{Kind: monitoringv1.PodMonitorsKind, APIVersion: MonitoringAPIVersion}, + ObjectMeta: metav1.ObjectMeta{ + Name: TigeraGatewayPodMonitor, + Namespace: common.TigeraPrometheusNamespace, + Labels: map[string]string{ + "team": "network-operators", + }, + }, + Spec: monitoringv1.PodMonitorSpec{ + Selector: metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app.kubernetes.io/component": "proxy", + "app.kubernetes.io/managed-by": "envoy-gateway", + }, + }, + NamespaceSelector: monitoringv1.NamespaceSelector{ + MatchNames: []string{TigeraGatewayNamespace}, + }, + PodMetricsEndpoints: []monitoringv1.PodMetricsEndpoint{ + { + Port: "metrics", + ScrapeTimeout: "5s", + Interval: "5s", + Scheme: "http", + HonorLabels: true, + }, + }, + }, + } +} + func (mc *monitorComponent) serviceMonitorCalicoNode() *monitoringv1.ServiceMonitor { endpoints := []monitoringv1.Endpoint{ { @@ -1204,6 +1275,14 @@ func allowTigeraPrometheusPolicy(cfg *Config) *v3.NetworkPolicy { Ports: networkpolicy.Ports(AlertmanagerPort), }, }, + { + Action: v3.Allow, + Protocol: &networkpolicy.TCPProtocol, + Destination: v3.EntityRule{ + // Egress access for Gateway Prometheus metrics + Ports: networkpolicy.Ports(TigeraGatewayComponentsMetricsPort), + }, + }, { Action: v3.Allow, Protocol: &networkpolicy.TCPProtocol, From 78af42cc4b05bc7e0a49298c21db0e2536c5df1c Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Thu, 3 Apr 2025 23:49:55 +0100 Subject: [PATCH 3/4] fix log collector missing image --- pkg/render/gateway_api.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/render/gateway_api.go b/pkg/render/gateway_api.go index feb7b374c8..7d4e113aa6 100644 --- a/pkg/render/gateway_api.go +++ b/pkg/render/gateway_api.go @@ -382,6 +382,12 @@ func (pr *gatewayAPIImplementationComponent) ResolveImages(is *operatorv1.ImageS return err } } + + pr.logCollectorImage, err = components.GetReference(components.ComponentL7Collector, reg, path, prefix, is) + if err != nil { + return err + } + return nil } @@ -575,6 +581,7 @@ func (pr *gatewayAPIImplementationComponent) logCollectorDeploymentAndService() return []client.Object{ // The log collector deployment. &appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{Kind: "Deployment", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "tigera-l7-log-collector", Namespace: EnvoyGatewayNamespace, @@ -603,6 +610,7 @@ func (pr *gatewayAPIImplementationComponent) logCollectorDeploymentAndService() }, // The log collector service. &corev1.Service{ + TypeMeta: metav1.TypeMeta{Kind: "Service", APIVersion: "v1"}, ObjectMeta: metav1.ObjectMeta{ Name: "tigera-l7-log-collector-service", Namespace: EnvoyGatewayNamespace, From f68240a6c096eca796db4e498b2110b958534809 Mon Sep 17 00:00:00 2001 From: Seth Malaki Date: Fri, 4 Apr 2025 18:49:51 +0100 Subject: [PATCH 4/4] fix service/pod monitor --- pkg/render/gateway_api.go | 2 +- pkg/render/monitor/monitor.go | 25 +++++++++++++------------ 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/pkg/render/gateway_api.go b/pkg/render/gateway_api.go index 7d4e113aa6..2b70d66be0 100644 --- a/pkg/render/gateway_api.go +++ b/pkg/render/gateway_api.go @@ -560,7 +560,7 @@ func (pr *gatewayAPIImplementationComponent) logCollectorVolumes() []corev1.Volu func (pr *gatewayAPIImplementationComponent) logCollectorEnv() []corev1.EnvVar { envs := []corev1.EnvVar{ - {Name: "LOG_LEVEL", Value: "Info"}, + {Name: "LOG_LEVEL", Value: "Debug"}, {Name: "FELIX_DIAL_TARGET", Value: "/var/run/felix/nodeagent/socket"}, // only use ALS gRPC endpoint {Name: "LISTEN_ADDRESS", Value: ":8080"}, diff --git a/pkg/render/monitor/monitor.go b/pkg/render/monitor/monitor.go index 699cdbfa23..913ced4e81 100644 --- a/pkg/render/monitor/monitor.go +++ b/pkg/render/monitor/monitor.go @@ -91,7 +91,8 @@ const ( bearerTokenFile = "/var/run/secrets/kubernetes.io/serviceaccount/token" KubeControllerMetrics = "calico-kube-controllers-metrics" - TigeraGatewayPodMonitor = "tigera-gateway-pod-monitor" + TigeraGatewayServiceMonitor = "tigera-gateway-service-monitor" + TigeraGatewayPodsProxiesMonitor = "tigera-gateway-pod-proxies-monitor" TigeraGatewayComponentsMetricsPort = 19001 TigeraGatewayNamespace = "tigera-gateway" ) @@ -227,7 +228,7 @@ func (mc *monitorComponent) Objects() ([]client.Object, []client.Object) { mc.serviceMonitorFluentd(), mc.serviceMonitorQueryServer(), mc.serviceMonitorCalicoKubeControllers(), - mc.podMonitorTigeraGatewayAPIController(), + mc.serviceMonitorTigeraGatewayAPIController(), mc.podMonitorTigeraGatewayGateways(), ) @@ -823,26 +824,26 @@ func (mc *monitorComponent) prometheusRule() *monitoringv1.PrometheusRule { } } -func (mc *monitorComponent) podMonitorTigeraGatewayAPIController() *monitoringv1.PodMonitor { - return &monitoringv1.PodMonitor{ - TypeMeta: metav1.TypeMeta{Kind: monitoringv1.PodMonitorsKind, APIVersion: MonitoringAPIVersion}, +func (mc *monitorComponent) serviceMonitorTigeraGatewayAPIController() *monitoringv1.ServiceMonitor { + return &monitoringv1.ServiceMonitor{ + TypeMeta: metav1.TypeMeta{Kind: monitoringv1.ServiceMonitorsKind, APIVersion: MonitoringAPIVersion}, ObjectMeta: metav1.ObjectMeta{ - Name: TigeraGatewayPodMonitor, + Name: TigeraGatewayServiceMonitor, Namespace: common.TigeraPrometheusNamespace, Labels: map[string]string{ "team": "network-operators", }, }, - Spec: monitoringv1.PodMonitorSpec{ + Spec: monitoringv1.ServiceMonitorSpec{ Selector: metav1.LabelSelector{ MatchLabels: map[string]string{ - "k8s-app": "gateway-api-controller", + "app.kubernetes.io/instance": "tigera-gateway-api", }, }, NamespaceSelector: monitoringv1.NamespaceSelector{ MatchNames: []string{TigeraGatewayNamespace}, }, - PodMetricsEndpoints: []monitoringv1.PodMetricsEndpoint{ + Endpoints: []monitoringv1.Endpoint{ { Port: "metrics", ScrapeTimeout: "5s", @@ -859,7 +860,7 @@ func (mc *monitorComponent) podMonitorTigeraGatewayGateways() *monitoringv1.PodM return &monitoringv1.PodMonitor{ TypeMeta: metav1.TypeMeta{Kind: monitoringv1.PodMonitorsKind, APIVersion: MonitoringAPIVersion}, ObjectMeta: metav1.ObjectMeta{ - Name: TigeraGatewayPodMonitor, + Name: TigeraGatewayPodsProxiesMonitor, Namespace: common.TigeraPrometheusNamespace, Labels: map[string]string{ "team": "network-operators", @@ -868,8 +869,7 @@ func (mc *monitorComponent) podMonitorTigeraGatewayGateways() *monitoringv1.PodM Spec: monitoringv1.PodMonitorSpec{ Selector: metav1.LabelSelector{ MatchLabels: map[string]string{ - "app.kubernetes.io/component": "proxy", - "app.kubernetes.io/managed-by": "envoy-gateway", + "app.kubernetes.io/component": "proxy", }, }, NamespaceSelector: monitoringv1.NamespaceSelector{ @@ -882,6 +882,7 @@ func (mc *monitorComponent) podMonitorTigeraGatewayGateways() *monitoringv1.PodM Interval: "5s", Scheme: "http", HonorLabels: true, + Path: "/stats/prometheus", }, }, },