https://rancher.com/ logo
Join Slack
Powered by
This message was deleted.
# harvester
a
This message was deleted.
b
You should be able to log into the nodes and see the IPs there
ip -br -c a
?
You should see something like 
stornet-bo
and 
stornet-br.101@stornet-br
with an IP I think. from there, just try to ping the other nodes via the 172.16.0.x addresses
f
Harvester node 3 --- the one that has the issues
b
What do the working nodes look like?
f
Harvester node 2 --- a healthy one
Harvester node 1 --- the main one
b
I'm not sure what's up with that ovs-system, but it shouldn't be related (at least for now)
Next I'd check that the switch sees the LAG up for node 3.
And verify that VLAN 101 is on the LAG and tagged traffic.
OH wait a minute, the 
storenet-br
should probably be showing UP and not Unknown.
like how the mgmt-br is 
UP
Maybe try to use wicked to bounce it?
f
in all the nodes
b
if it helps we have this script for verify the longhorn pods can all talk to each other on storage network including an MTU check
Copy code
import subprocess
import os
import time

# Verify all harvester nodes can communicate to all other harvester nodes over the storage network.
# Automation of step 4 from <https://docs.harvesterhci.io/v1.3/advanced/storagenetwork#step-4>
# Requires kubectl and an active context for the harvester cluster.
# Can be run on a harvester management node.

# Usage python3 ./verify-storage-network.py

MTU = int(os.environ.get("MTU", 9000))


def main():
    print("Getting all instance manager pods")
    pod_names_cmd = r"""kubectl get pods -n longhorn-system -l <http://longhorn.io/component=instance-manager|longhorn.io/component=instance-manager> -o=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}'"""
    result = subprocess.run(pod_names_cmd, shell=True,
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
    pod_names = [x.decode("utf-8") for x in result.stdout.splitlines()]
    print(pod_names)

    print("Getting all instance manager nodes")
    pod_nodes_cmd = r"""kubectl get pods -n longhorn-system -l <http://longhorn.io/component=instance-manager|longhorn.io/component=instance-manager> -o=jsonpath='{range .items[*]}{.spec.nodeName}{"\n"}{end}'"""
    result = subprocess.run(pod_nodes_cmd, shell=True,
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
    pod_nodes = [x.decode("utf-8") for x in result.stdout.splitlines()]
    print(pod_nodes)

    print("Getting all instance manager pod IPs")
    pod_ips_cmd = r"""kubectl get pods -n longhorn-system -l <http://longhorn.io/component=instance-manager|longhorn.io/component=instance-manager> -o jsonpath='{.items[*].metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq '.[] | select(.name | startswith("harvester-system")) | .ips[0]' -r"""
    result = subprocess.run(
        pod_ips_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
    pod_ips = result.stdout.decode("utf-8").splitlines()
    print(pod_ips)

    print("Getting all instance manager pod MACs")
    pod_macs_cmd = r"""kubectl get pods -n longhorn-system -l <http://longhorn.io/component=instance-manager|longhorn.io/component=instance-manager> -o jsonpath='{.items[*].metadata.annotations.k8s\.v1\.cni\.cncf\.io/network-status}' | jq '.[] | select(.name | startswith("harvester-system")) | .mac' -r"""
    result = subprocess.run(pod_macs_cmd, shell=True,
                            stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
    pod_macs = result.stdout.decode("utf-8").splitlines()
    print(pod_macs)

    print("\nDebug info")
    for idx, pod_name in enumerate(pod_names):
        print(f"{pod_name}\t{pod_macs[idx]}\t{pod_ips[idx]}\t{pod_nodes[idx]}")

    print("\nStarting webservers")
    processes = []
    for idx, pod_name in enumerate(pod_names):
        pod_ip = pod_ips[idx]
        cmd = f"""kubectl exec -n longhorn-system {
            pod_name} -- python3 -m http.server 8000 --bind {pod_ip}"""
        print("Running cmd:", cmd)
        processes.append(subprocess.Popen(
            cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE))

    print("\nPausing for webserver start")
    time.sleep(6)

    print("\nVerifying connectivity between webservers")
    for pod_idx, pod_name in enumerate(pod_names):
        for ip_idx, ip in enumerate(pod_ips):
            pod_curl_cmd = f"""kubectl exec -n longhorn-system {
                pod_name} -- curl {ip}:8000 -m 1"""
            print(f"Checking {pod_nodes[pod_idx]} <-> {pod_nodes[ip_idx]}")
            try:
                result = subprocess.run(
                    pod_curl_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
                result.stdout.decode("utf-8").splitlines()
            except subprocess.CalledProcessError:
                print("Connection failed")

    print("\nStopping webservers")
    for proc in processes:
        proc.kill()
    for pod_name in pod_names:
        print(f"Killing webserver on {pod_name}")
        pod_kill_cmd = f"""kubectl exec -n longhorn-system {
            pod_name} -- pkill -f 'python3 -m http.server 8000'"""
        subprocess.run(pod_kill_cmd, shell=True, stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE, check=False)

    print("\nCopying ping")
    processes = []
    for idx, pod_name in enumerate(pod_names):
        cmd = f"""kubectl -n longhorn-system cp $(which ping) {
            pod_name}:/tmp/ping"""
        print("Running cmd:", cmd)
        processes.append(subprocess.Popen(
            cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE))

    print(f"\nChecking MTU {MTU}")
    print(f"An instance can sometimes fail to ping itself")
    for pod_idx, pod_name in enumerate(pod_names):
        for ip_idx, ip in enumerate(pod_ips):
            pod_curl_cmd = f"""kubectl exec -n longhorn-system {
                pod_name} -- /tmp/ping -c 4 -w 4 -M do -s {MTU-28} {ip} -t 4"""  # 28 for packet headers in ping
            print(f"Checking {pod_nodes[pod_idx]} <-> {pod_nodes[ip_idx]}")
            try:
                result = subprocess.run(
                    pod_curl_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=True)
                result.stdout.decode("utf-8").splitlines()
            except subprocess.CalledProcessError:
                print("Connection failed")

    print("\nRemoving ping")
    for pod_name in pod_names:
        print(f"Removing ping on {pod_name}")
        pod_kill_cmd = f"""kubectl exec -n longhorn-system {
            pod_name} -- rm /tmp/ping"""
        subprocess.run(pod_kill_cmd, shell=True, stdout=subprocess.PIPE,
                       stderr=subprocess.PIPE, check=False)

    print("\nFinished")


if __name__ == "__main__":
    main()
☝️ 1
b
I'd recommend that script check
I have a bash one, but it's not designed for selecting the storage network
f
thanks Guys
let me run that and come back
b
Maybe @brainy-kilobyte-33711 can advise how his lags (if any) look. We simplified and lagged the mgmt network and break off vlans from that so it's weird.
f
Copy code
harvester-node03:/tmp # python3 test.py 
Getting all instance manager pods
['instance-manager-5b16e4008734c79656437fa2b7d1b65b', 'instance-manager-6c85f04c09567361783a3bf42851f646', 'instance-manager-ed2dbd01100fa0f8a629a7dd31e3f9f3']
Getting all instance manager nodes
['harvester-node01', 'harvester-node02', 'harvester-node03']
Getting all instance manager pod IPs
['172.16.0.1', '172.16.0.2', '172.16.0.5']
Getting all instance manager pod MACs
['1e:03:29:74:72:41', 'ba:77:f2:38:5c:93', '9a:28:f8:6f:e8:05']

Debug info
instance-manager-5b16e4008734c79656437fa2b7d1b65b	1e:03:29:74:72:41	172.16.0.1	harvester-node01
instance-manager-6c85f04c09567361783a3bf42851f646	ba:77:f2:38:5c:93	172.16.0.2	harvester-node02
instance-manager-ed2dbd01100fa0f8a629a7dd31e3f9f3	9a:28:f8:6f:e8:05	172.16.0.5	harvester-node03

Starting webservers
Running cmd: kubectl exec -n longhorn-system instance-manager-5b16e4008734c79656437fa2b7d1b65b -- python3 -m http.server 8000 --bind 172.16.0.1
Running cmd: kubectl exec -n longhorn-system instance-manager-6c85f04c09567361783a3bf42851f646 -- python3 -m http.server 8000 --bind 172.16.0.2
Running cmd: kubectl exec -n longhorn-system instance-manager-ed2dbd01100fa0f8a629a7dd31e3f9f3 -- python3 -m http.server 8000 --bind 172.16.0.5

Pausing for webserver start

Verifying connectivity between webservers
Checking harvester-node01 <-> harvester-node01
Checking harvester-node01 <-> harvester-node02
Checking harvester-node01 <-> harvester-node03
Checking harvester-node02 <-> harvester-node01
Checking harvester-node02 <-> harvester-node02
Checking harvester-node02 <-> harvester-node03
Checking harvester-node03 <-> harvester-node01
Checking harvester-node03 <-> harvester-node02
Checking harvester-node03 <-> harvester-node03

Stopping webservers
Killing webserver on instance-manager-5b16e4008734c79656437fa2b7d1b65b
Killing webserver on instance-manager-6c85f04c09567361783a3bf42851f646
Killing webserver on instance-manager-ed2dbd01100fa0f8a629a7dd31e3f9f3

Copying ping
Running cmd: kubectl -n longhorn-system cp $(which ping) instance-manager-5b16e4008734c79656437fa2b7d1b65b:/tmp/ping
Running cmd: kubectl -n longhorn-system cp $(which ping) instance-manager-6c85f04c09567361783a3bf42851f646:/tmp/ping
Running cmd: kubectl -n longhorn-system cp $(which ping) instance-manager-ed2dbd01100fa0f8a629a7dd31e3f9f3:/tmp/ping

Checking MTU 9000
An instance can sometimes fail to ping itself
Checking harvester-node01 <-> harvester-node01
Connection failed
Checking harvester-node01 <-> harvester-node02
Connection failed
Checking harvester-node01 <-> harvester-node03
Connection failed
Checking harvester-node02 <-> harvester-node01
Connection failed
Checking harvester-node02 <-> harvester-node02
Checking harvester-node02 <-> harvester-node03
Connection failed
Checking harvester-node03 <-> harvester-node01
Connection failed
Checking harvester-node03 <-> harvester-node02
Connection failed
Checking harvester-node03 <-> harvester-node03

Removing ping
Removing ping on instance-manager-5b16e4008734c79656437fa2b7d1b65b
Removing ping on instance-manager-6c85f04c09567361783a3bf42851f646
Removing ping on instance-manager-ed2dbd01100fa0f8a629a7dd31e3f9f3

Finished
harvester-node03:/tmp #
im gussing this goes over some NAT inside Harvester
but idk at this point im lost
b
sorry I am not in a place with access to the cluster right now, can have a look monday
f
Thanks I would appreciate it
Issue solved network config @few-appointment-23216 managed to solve the issue in the harvester-node03 the longhorn was failing to create volume replicas in the node the
Copy code
/var/lib/harvester/defaultdisk/replicas
directory was missing I created it manually and the issue was fixed
b
Are you using a separate data disk? Seems odd that dir isn't there
f
yes
we are using 1 ssd for the harvester os and some other ssd for data disk configured as raid0
the issue was only with node 3
and now its deleting some vm volumes
its supper wired
i think im missing some more dirs
a
AIUI longhorn should have created the 
replicas
subdirectory by itself. Uh... Just checking, is your data disk mounted on 
/var/lib/harvester/defaultdisk
? What does 
mount | grep defaultdisk
output? Does this show the expected disk mounted at that location?
b
is the default disk being dev/sdb the same on your other nodes?
f
yes
b
and just to triple check if you run 
mountpoint /var/lib/harvester/defaultdisk
it says it's a mountpoint?
could you run that on node 3 the problematic one
b
Your network tests are still pasing for node3 as well?
b
very weird, don't know what else to suggest apart from creating an issue with a support bundle so someone can log at the logs
f
network looks ok to me
thanks a lot @brainy-kilobyte-33711
the volumes now are ok
i just lost 5 vms
b
You might look under orphaned replicas
f
their disks are no where to be found lol
b
You might be able to regen the disks from there.
f
i will look into it thanks @bland-article-62755
Open in Slack
PreviousNext
Powered by