As you know, the new version of dataplane v2 uses cilium with eBPF.
Unfortunately, Topology Aware Routing does not work in the current implementation. For example, I fulfill all the minimum necessary conditions to implement this function, but in cilium bpf lb list I see all possible backend addresses, including addresses from other zones.
therefore, from the Cilium documentation, it is enough to enable the parameter loadBalancer.serviceTopology=true
but how to do it in GKE, where the configuration data is closed for viewing and setting, is not entirely clear
To add details, I deployed a test cluster with two nodes located in different zones:
y.huzii@yhuzii:~/test$ k get nodes
NAME STATUS ROLES AGE VERSION
gke-cluster-2-default-pool-482e17e2-b5pd Ready <none> 16m v1.28.9-gke.1289002
gke-cluster-2-default-pool-4bf46657-z4l0 Ready <none> 16m v1.28.9-gke.1289002
y.huzii@yhuzii:~/test$ k get nodes -l topology.kubernetes.io/zone=europe-central2-b
NAME STATUS ROLES AGE VERSION
gke-cluster-2-default-pool-482e17e2-b5pd Ready <none> 17m v1.28.9-gke.1289002
y.huzii@yhuzii:~/test$ k get nodes -l topology.kubernetes.io/zone=europe-central2-a
NAME STATUS ROLES AGE VERSION
gke-cluster-2-default-pool-4bf46657-z4l0 Ready <none> 17m v1.28.9-gke.1289002
created a deployment of 4 pods, where it is clearly visible that 2 pods fell on each of the nodes:
y.huzii@yhuzii:~/test$ k get pods -o wide -l app=nginx
NAME READY STATUS RESTARTS AGE IP NODE NOMINATED NODE READINESS GATES
nginx-5455fc8796-lnsvc 1/1 Running 0 14m 10.16.1.8 gke-cluster-2-default-pool-482e17e2-b5pd <none> <none>
nginx-5455fc8796-lxxpp 1/1 Running 0 14m 10.16.0.14 gke-cluster-2-default-pool-4bf46657-z4l0 <none> <none>
nginx-5455fc8796-n9mmd 1/1 Running 0 14m 10.16.0.13 gke-cluster-2-default-pool-4bf46657-z4l0 <none> <none>
nginx-5455fc8796-wp5xd 1/1 Running 0 14m 10.16.1.7 gke-cluster-2-default-pool-482e17e2-b5pd <none> <none>
from the description of the service, it can be seen that an annotation was added for Topology Aware Routing:
y.huzii@yhuzii:~/test$ k describe service nginx
Name: nginx
Namespace: default
Labels: <none>
Annotations: cloud.google.com/neg: {"ingress":true}
service.kubernetes.io/topology-mode: auto
Selector: app=nginx
Type: ClusterIP
IP Family Policy: SingleStack
IP Families: IPv4
IP: 10.3.225.126
IPs: 10.3.225.126
Port: <unset> 80/TCP
TargetPort: 80/TCP
Endpoints: 10.16.0.13:80,10.16.0.14:80,10.16.1.7:80 + 1 more...
Session Affinity: None
Events: <none>
the appearance of hints confirms this:
y.huzii@yhuzii:~/test$ k get EndpointSlice nginx-bbgr9 -o yaml
addressType: IPv4
apiVersion: discovery.k8s.io/v1
endpoints:
- addresses:
- 10.16.1.7
conditions:
ready: true
serving: true
terminating: false
hints:
forZones:
- name: europe-central2-b
nodeName: gke-cluster-2-default-pool-482e17e2-b5pd
targetRef:
kind: Pod
name: nginx-5455fc8796-wp5xd
namespace: default
uid: 9e18f1f5-cab2-498b-acbd-2565fd2a5d58
zone: europe-central2-b
- addresses:
- 10.16.0.14
conditions:
ready: true
serving: true
terminating: false
hints:
forZones:
- name: europe-central2-a
nodeName: gke-cluster-2-default-pool-4bf46657-z4l0
targetRef:
kind: Pod
name: nginx-5455fc8796-lxxpp
namespace: default
uid: 5eddc904-542b-4ec2-8a38-4ec783b79f59
zone: europe-central2-a
- addresses:
- 10.16.0.13
conditions:
ready: true
serving: true
terminating: false
hints:
forZones:
- name: europe-central2-a
nodeName: gke-cluster-2-default-pool-4bf46657-z4l0
targetRef:
kind: Pod
name: nginx-5455fc8796-n9mmd
namespace: default
uid: 7ed8696e-d7d7-47fa-89dc-b78a9c45254a
zone: europe-central2-a
- addresses:
- 10.16.1.8
conditions:
ready: true
serving: true
terminating: false
hints:
forZones:
- name: europe-central2-b
nodeName: gke-cluster-2-default-pool-482e17e2-b5pd
targetRef:
kind: Pod
name: nginx-5455fc8796-lnsvc
namespace: default
uid: 87f99df1-21ee-4310-8a5c-2c60cb07f205
zone: europe-central2-b
kind: EndpointSlice
metadata:
annotations:
endpoints.kubernetes.io/last-change-trigger-time: "2024-07-18T08:10:11Z"
creationTimestamp: "2024-07-18T08:10:11Z"
generateName: nginx-
generation: 1
labels:
endpointslice.kubernetes.io/managed-by: endpointslice-controller.k8s.io
kubernetes.io/service-name: nginx
name: nginx-bbgr9
namespace: default
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Service
name: nginx
uid: ae157e96-1609-4d4e-adac-744c84794433
resourceVersion: "16669"
uid: 5183db7c-1d34-46f4-b3a9-7f59928632da
ports:
- name: ""
port: 80
protocol: TCP
but in the cilium agent pod, on one of the nodes I see that the service has 4 backends, instead of the expected two:
root@gke-cluster-2-default-pool-482e17e2-b5pd:/home/cilium# cilium bpf lb list | grep 10.3.225.126 -A 4
10.3.225.126:80 10.16.0.13:80 (14) (2)
10.16.0.14:80 (14) (1)
10.16.1.8:80 (14) (3)
10.16.1.7:80 (14) (4)
0.0.0.0:0 (14) (0) [ClusterIP, non-routable]
the required parameter in cilium is enabled, I see it in the debug:
root@gke-cluster-2-default-pool-482e17e2-b5pd:/home/cilium# cilium debuginfo | grep enable-service-topology
enable-service-topology:true
accordingly, when creating a test source of traffic in one of the zones, instead of the expected 2 backends, I see that all possible 4 backends apply, including the pods from the second zone:
apiVersion: apps/v1
kind: Deployment
metadata:
name: curl
spec:
replicas: 1
selector:
matchLabels:
app: curl
template:
metadata:
creationTimestamp: null
labels:
app: curl
spec:
topologySpreadConstraints:
- maxSkew: 1
topologyKey: topology.kubernetes.io/zone
whenUnsatisfiable: DoNotSchedule
labelSelector:
matchLabels:
app: curl
containers:
- name: curl
image: docker.io/curlimages/curl:latest
args:
- sh
- -c
- 'while true; do curl -s -v nginx 2>&1 | grep X-Server-Hostname; sleep 1; done;'
y.huzii@yhuzii:~/test$ kubectl logs curl-5fcbf7c896-g2rvw | cut -f 2 -d ':' | sort | uniq -c
423 nginx-5455fc8796-lnsvc
430 nginx-5455fc8796-lxxpp
415 nginx-5455fc8796-n9mmd
384 nginx-5455fc8796-wp5xd
LinkedIn
Twitter
