~Hello All,
We are facing issue in Downstream cluster when we update Rancher.
Our setup is:
Rancher Version: 2.6.3
Local cluster: K3s 1.23.9
Downstream cluster (on bare-metal) using RKE: K8s 1.21.6
When we are trying to update Rancher from version 2.6.3 to 2.6.6, Cluster agent pod in Downstream cluster (cattle-cluster-agent) is going into crashloopbackoff status.
And we see below erros in cattle-cluster-agent pod:
starting /v1, Kind=Secret controller"
E0804 09:55:45.612622 39 runtime.go:78] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference)
goroutine 4699 [running]:
k8s.io/apimachinery/pkg/util/runtime.logPanic({0x37e9880, 0x6ea0100})
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/runtime/runtime.go:74 +0x7d
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc0099ef5f0})
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/runtime/runtime.go:48 +0x75
panic({0x37e9880, 0x6ea0100})
/usr/lib64/go/1.17/src/runtime/panic.go:1038 +0x215
github.com/rancher/dynamiclistener/factory.(*TLS).Merge(0x34, 0x0, 0xc00c117c80)
/mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/factory/gen.go:79 +0x70
github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).saveInK8s(0xc0001f2c40, 0xc00c117c80)
/mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:134 +0x96
github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).Update(0xc0001f2c40, 0xc009943b70)
/mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:169 +0xb3
github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).init.func1({0x0, 0x0}, 0xc00c117c80)
/mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:83 +0xa5
github.com/rancher/wrangler/pkg/generated/controllers/core/v1.FromSecretHandlerToHandler.func1({0xc00bff0360, 0x1e14d34f513eba}, {0x47b3720, 0xc00c117c80})
/mailto:go/pkg/mod/github.com/rancher/wrangler@v0.8.11-0.20220411195911-c2b951ab3480/pkg/generated/controllers/core/v1/secret.go:102 +0x44
github.com/rancher/lasso/pkg/controller.SharedControllerHandlerFunc.OnChange(0xc0099ef5e0, {0xc00bff0360, 0xc010633080}, {0x47b3720, 0xc00c117c80})
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/sharedcontroller.go:29 +0x38
github.com/rancher/lasso/pkg/controller.(*SharedHandler).OnChange(0xc000cc9ae0, {0xc00bff0360, 0x1a}, {0x47b3720, 0xc00c117c80})
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/sharedhandler.go:75 +0x23f
github.com/rancher/lasso/pkg/controller.(*controller).syncHandler(0xc000d749a0, {0xc00bff0360, 0x1a})
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:220 +0x93
github.com/rancher/lasso/pkg/controller.(*controller).processSingleItem(0xc000d749a0, {0x35b2e60, 0xc0099ef5f0})
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:201 +0x10e
github.com/rancher/lasso/pkg/controller.(*controller).processNextWorkItem(0xc000d749a0)
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:178 +0x46
github.com/rancher/lasso/pkg/controller.(*controller).runWorker(...)
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:167
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fbb1a657fe8)
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0018db500, {0x4781a40, 0xc00f15a270}, 0x1, 0xc000cf5f80)
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0x80, 0x440c45)
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(0xc00978e758, 0xc00978e778, 0xc00978e768)
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:90 +0x25
created by
github.com/rancher/lasso/pkg/controller.(*controller).run
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:135 +0x2c6
panic: runtime error: invalid memory address or nil pointer dereference [recovered]
panic: runtime error: invalid memory address or nil pointer dereference
[signal SIGSEGV: segmentation violation code=0x1 addr=0xb8 pc=0x2168f10]
goroutine 4699 [running]:
k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc0099ef5f0})
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/runtime/runtime.go:55 +0xd8
panic({0x37e9880, 0x6ea0100})
/usr/lib64/go/1.17/src/runtime/panic.go:1038 +0x215
github.com/rancher/dynamiclistener/factory.(*TLS).Merge(0x34, 0x0, 0xc00c117c80)
/mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/factory/gen.go:79 +0x70
github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).saveInK8s(0xc0001f2c40, 0xc00c117c80)
/mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:134 +0x96
github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).Update(0xc0001f2c40, 0xc009943b70)
/mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:169 +0xb3
github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).init.func1({0x0, 0x0}, 0xc00c117c80)
/mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:83 +0xa5
github.com/rancher/wrangler/pkg/generated/controllers/core/v1.FromSecretHandlerToHandler.func1({0xc00bff0360, 0x1e14d34f513eba}, {0x47b3720, 0xc00c117c80})
/mailto:go/pkg/mod/github.com/rancher/wrangler@v0.8.11-0.20220411195911-c2b951ab3480/pkg/generated/controllers/core/v1/secret.go:102 +0x44
github.com/rancher/lasso/pkg/controller.SharedControllerHandlerFunc.OnChange(0xc0099ef5e0, {0xc00bff0360, 0xc010633080}, {0x47b3720, 0xc00c117c80})
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/sharedcontroller.go:29 +0x38
github.com/rancher/lasso/pkg/controller.(*SharedHandler).OnChange(0xc000cc9ae0, {0xc00bff0360, 0x1a}, {0x47b3720, 0xc00c117c80})
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/sharedhandler.go:75 +0x23f
github.com/rancher/lasso/pkg/controller.(*controller).syncHandler(0xc000d749a0, {0xc00bff0360, 0x1a})
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:220 +0x93
github.com/rancher/lasso/pkg/controller.(*controller).processSingleItem(0xc000d749a0, {0x35b2e60, 0xc0099ef5f0})
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:201 +0x10e
github.com/rancher/lasso/pkg/controller.(*controller).processNextWorkItem(0xc000d749a0)
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:178 +0x46
github.com/rancher/lasso/pkg/controller.(*controller).runWorker(...)
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:167
k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fbb1a657fe8)
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:155 +0x67
k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0018db500, {0x4781a40, 0xc00f15a270}, 0x1, 0xc000cf5f80)
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:156 +0xb6
k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0x80, 0x440c45)
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:133 +0x89
k8s.io/apimachinery/pkg/util/wait.Until(0xc00978e758, 0xc00978e778, 0xc00978e768)
/mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:90 +0x25
created by
github.com/rancher/lasso/pkg/controller.(*controller).run
/mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:135 +0x2c6
Can someone please help us here?
Thank you~