~Hello All, We are facing issue in Downstream cluster when we update Rancher. Our setup is: Rancher Version: 2.6.3 Local cluster: K3s 1.23.9 Downstream cluster (on bare-metal) using RKE: K8s 1.21.6 When we are trying to update Rancher from version 2.6.3 to 2.6.6, Cluster agent pod in Downstream cluster (cattle-cluster-agent) is going into crashloopbackoff status. And we see below erros in cattle-cluster-agent pod: starting /v1, Kind=Secret controller" E0804 095545.612622 39 runtime.go:78] Observed a panic: "invalid memory address or nil pointer dereference" (runtime error: invalid memory address or nil pointer dereference) goroutine 4699 [running]: k8s.io/apimachinery/pkg/util/runtime.logPanic({0x37e9880, 0x6ea0100}) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/runtime/runtime.go:74 +0x7d k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc0099ef5f0}) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/runtime/runtime.go:48 +0x75 panic({0x37e9880, 0x6ea0100}) /usr/lib64/go/1.17/src/runtime/panic.go:1038 +0x215 github.com/rancher/dynamiclistener/factory.(*TLS).Merge(0x34, 0x0, 0xc00c117c80) /mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/factory/gen.go:79 +0x70 github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).saveInK8s(0xc0001f2c40, 0xc00c117c80) /mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:134 +0x96 github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).Update(0xc0001f2c40, 0xc009943b70) /mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:169 +0xb3 github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).init.func1({0x0, 0x0}, 0xc00c117c80) /mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:83 +0xa5 github.com/rancher/wrangler/pkg/generated/controllers/core/v1.FromSecretHandlerToHandler.func1({0xc00bff0360, 0x1e14d34f513eba}, {0x47b3720, 0xc00c117c80}) /mailto:go/pkg/mod/github.com/rancher/wrangler@v0.8.11-0.20220411195911-c2b951ab3480/pkg/generated/controllers/core/v1/secret.go:102 +0x44 github.com/rancher/lasso/pkg/controller.SharedControllerHandlerFunc.OnChange(0xc0099ef5e0, {0xc00bff0360, 0xc010633080}, {0x47b3720, 0xc00c117c80}) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/sharedcontroller.go:29 +0x38 github.com/rancher/lasso/pkg/controller.(*SharedHandler).OnChange(0xc000cc9ae0, {0xc00bff0360, 0x1a}, {0x47b3720, 0xc00c117c80}) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/sharedhandler.go:75 +0x23f github.com/rancher/lasso/pkg/controller.(*controller).syncHandler(0xc000d749a0, {0xc00bff0360, 0x1a}) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:220 +0x93 github.com/rancher/lasso/pkg/controller.(*controller).processSingleItem(0xc000d749a0, {0x35b2e60, 0xc0099ef5f0}) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:201 +0x10e github.com/rancher/lasso/pkg/controller.(*controller).processNextWorkItem(0xc000d749a0) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:178 +0x46 github.com/rancher/lasso/pkg/controller.(*controller).runWorker(...) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:167 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fbb1a657fe8) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:155 +0x67 k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0018db500, {0x4781a40, 0xc00f15a270}, 0x1, 0xc000cf5f80) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0x80, 0x440c45) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(0xc00978e758, 0xc00978e778, 0xc00978e768) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:90 +0x25 created by github.com/rancher/lasso/pkg/controller.(*controller).run /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:135 +0x2c6 panic: runtime error: invalid memory address or nil pointer dereference [recovered] panic: runtime error: invalid memory address or nil pointer dereference [signal SIGSEGV: segmentation violation code=0x1 addr=0xb8 pc=0x2168f10] goroutine 4699 [running]: k8s.io/apimachinery/pkg/util/runtime.HandleCrash({0x0, 0x0, 0xc0099ef5f0}) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/runtime/runtime.go:55 +0xd8 panic({0x37e9880, 0x6ea0100}) /usr/lib64/go/1.17/src/runtime/panic.go:1038 +0x215 github.com/rancher/dynamiclistener/factory.(*TLS).Merge(0x34, 0x0, 0xc00c117c80) /mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/factory/gen.go:79 +0x70 github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).saveInK8s(0xc0001f2c40, 0xc00c117c80) /mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:134 +0x96 github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).Update(0xc0001f2c40, 0xc009943b70) /mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:169 +0xb3 github.com/rancher/dynamiclistener/storage/kubernetes.(*storage).init.func1({0x0, 0x0}, 0xc00c117c80) /mailto:go/pkg/mod/github.com/rancher/dynamiclistener@v0.3.1-0.20210616080009-9865ae859c7f/storage/kubernetes/controller.go:83 +0xa5 github.com/rancher/wrangler/pkg/generated/controllers/core/v1.FromSecretHandlerToHandler.func1({0xc00bff0360, 0x1e14d34f513eba}, {0x47b3720, 0xc00c117c80}) /mailto:go/pkg/mod/github.com/rancher/wrangler@v0.8.11-0.20220411195911-c2b951ab3480/pkg/generated/controllers/core/v1/secret.go:102 +0x44 github.com/rancher/lasso/pkg/controller.SharedControllerHandlerFunc.OnChange(0xc0099ef5e0, {0xc00bff0360, 0xc010633080}, {0x47b3720, 0xc00c117c80}) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/sharedcontroller.go:29 +0x38 github.com/rancher/lasso/pkg/controller.(*SharedHandler).OnChange(0xc000cc9ae0, {0xc00bff0360, 0x1a}, {0x47b3720, 0xc00c117c80}) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/sharedhandler.go:75 +0x23f github.com/rancher/lasso/pkg/controller.(*controller).syncHandler(0xc000d749a0, {0xc00bff0360, 0x1a}) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:220 +0x93 github.com/rancher/lasso/pkg/controller.(*controller).processSingleItem(0xc000d749a0, {0x35b2e60, 0xc0099ef5f0}) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:201 +0x10e github.com/rancher/lasso/pkg/controller.(*controller).processNextWorkItem(0xc000d749a0) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:178 +0x46 github.com/rancher/lasso/pkg/controller.(*controller).runWorker(...) /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:167 k8s.io/apimachinery/pkg/util/wait.BackoffUntil.func1(0x7fbb1a657fe8) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:155 +0x67 k8s.io/apimachinery/pkg/util/wait.BackoffUntil(0xc0018db500, {0x4781a40, 0xc00f15a270}, 0x1, 0xc000cf5f80) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:156 +0xb6 k8s.io/apimachinery/pkg/util/wait.JitterUntil(0x0, 0x3b9aca00, 0x0, 0x80, 0x440c45) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:133 +0x89 k8s.io/apimachinery/pkg/util/wait.Until(0xc00978e758, 0xc00978e778, 0xc00978e768) /mailto:go/pkg/mod/k8s.io/apimachinery@v0.23.3/pkg/util/wait/wait.go:90 +0x25 created by github.com/rancher/lasso/pkg/controller.(*controller).run /mailto:go/pkg/mod/github.com/rancher/lasso@v0.0.0-20220412224715-5f3517291ad4/pkg/controller/controller.go:135 +0x2c6 Can someone please help us here? Thank you~