Any idea why my cilium-agent on k3s-agents is trying to connect to the api-server on https://127.0.0.1:6443? The 3 k3s-servers have all cilium pods running smooth but as soon as I join a k3s-agent in there it fails to start up cilium-agent there with this in the logs:

level=info msg="Initializing daemon" subsys=daemon
level=info msg="Establishing connection to apiserver" host="<https://127.0.0.1:6443>" subsys=k8s
level=info msg="Establishing connection to apiserver" host="<https://127.0.0.1:6443>" subsys=k8s
level=info msg="Establishing connection to apiserver" host="<https://127.0.0.1:6443>" subsys=k8s
level=info msg="Establishing connection to apiserver" host="<https://127.0.0.1:6443>" subsys=k8s

I've also tried using cilium CLI and set

cilium config set k8sServiceHost 192.168.250.11
cilium config set k8sServicePort 6443

but it keeps on trying to connect to 127.0.0.1:6443 I've disabled kube-proxy, network policy, servicelb and traefik during k3s install.

kubectl get nodes -A -o wide
NAME          STATUS   ROLES                       AGE   VERSION        INTERNAL-IP      EXTERNAL-IP   OS-IMAGE             KERNEL-VERSION      CONTAINER-RUNTIME
k3s-agent1    Ready    <none>                      8h    v1.24.6+k3s1   192.168.250.14   <none>        Ubuntu 20.04.5 LTS   5.4.0-128-generic   <containerd://1.6.8-k3s1>
k3s-server1   Ready    control-plane,etcd,master   9h    v1.24.6+k3s1   192.168.250.11   <none>        Ubuntu 20.04.5 LTS   5.4.0-128-generic   <containerd://1.6.8-k3s1>
k3s-server2   Ready    control-plane,etcd,master   9h    v1.24.6+k3s1   192.168.250.12   <none>        Ubuntu 20.04.5 LTS   5.4.0-128-generic   <containerd://1.6.8-k3s1>
k3s-server3   Ready    control-plane,etcd,master   8h    v1.24.6+k3s1   192.168.250.13   <none>        Ubuntu 20.04.5 LTS   5.4.0-128-generic   <containerd://1.6.8-k3s1>

root@k3s-server1:~# cilium status
    /¯¯\
 /¯¯\__/¯¯\    Cilium:         2 errors
 \__/¯¯\__/    Operator:       OK
 /¯¯\__/¯¯\    Hubble:         disabled
 \__/¯¯\__/    ClusterMesh:    disabled
    \__/

DaemonSet         cilium             Desired: 4, Ready: 3/4, Available: 3/4, Unavailable: 1/4
Deployment        cilium-operator    Desired: 1, Ready: 1/1, Available: 1/1
Containers:       cilium             Running: 4
                  cilium-operator    Running: 1
Cluster Pods:     3/3 managed by Cilium
Image versions    cilium             <http://quay.io/cilium/cilium:v1.12.2@sha256:986f8b04cfdb35cf714701e58e35da0ee63da2b8a048ab596ccb49de58d5ba36|quay.io/cilium/cilium:v1.12.2@sha256:986f8b04cfdb35cf714701e58e35da0ee63da2b8a048ab596ccb49de58d5ba36>: 4
                  cilium-operator    <http://quay.io/cilium/operator-generic:v1.12.2@sha256:00508f78dae5412161fa40ee30069c2802aef20f7bdd20e91423103ba8c0df6e|quay.io/cilium/operator-generic:v1.12.2@sha256:00508f78dae5412161fa40ee30069c2802aef20f7bdd20e91423103ba8c0df6e>: 1
Errors:           cilium             cilium          1 pods of DaemonSet cilium are not ready
                  cilium             cilium-4spdt    unable to retrieve cilium status: container cilium-agent is in CrashLoopBackOff, exited with code 1: level=fatal msg="Unable to initialize Kubernetes subsystem" error="unable to create k8s client: unable to create k8s client: Get \"<https://127.0.0.1:6443/api/v1/namespaces/kube-system>\": dial tcp 127.0.0.1:6443: connect: connection refused" subsys=daemon

root@k3s-server1:~# kubectl get pods -A -o wide
NAMESPACE     NAME                                      READY   STATUS             RESTARTS         AGE     IP               NODE          NOMINATED NODE   READINESS GATES
kube-system   cilium-4spdt                              0/1     CrashLoopBackOff   99 (2m25s ago)   7h15m   192.168.250.14   k3s-agent1    <none>           <none>
kube-system   cilium-6mjjd                              1/1     Running            1 (26m ago)      7h15m   192.168.250.13   k3s-server3   <none>           <none>
kube-system   cilium-9xcpk                              1/1     Running            1 (26m ago)      7h15m   192.168.250.11   k3s-server1   <none>           <none>
kube-system   cilium-b7grx                              1/1     Running            1 (25m ago)      7h15m   192.168.250.12   k3s-server2   <none>           <none>
kube-system   cilium-operator-7b5b55f786-4nfmd          1/1     Running            1 (25m ago)      9h      192.168.250.12   k3s-server2   <none>           <none>
kube-system   coredns-b96499967-96fsc                   1/1     Running            1 (25m ago)      9h      10.0.1.76        k3s-server2   <none>           <none>
kube-system   local-path-provisioner-7b7dc8d6f5-gbxjc   1/1     Running            1 (25m ago)      9h      10.0.1.20        k3s-server2   <none>           <none>
kube-system   metrics-server-668d979685-j6xzf           1/1     Running            1 (25m ago)      9h      10.0.1.177       k3s-server2   <none>           <none>