This message was deleted.

Here are our values, the backuptarget is null, not sure if that matters.

annotations: {}
csi:
  attacherReplicaCount: null
  kubeletRootDir: null
  provisionerReplicaCount: null
  resizerReplicaCount: null
  snapshotterReplicaCount: null
defaultSettings:
  allowCollectingLonghornUsageMetrics: null
  allowRecurringJobWhileVolumeDetached: null
  allowVolumeCreationWithDegradedAvailability: null
  autoCleanupSystemGeneratedSnapshot: null
  autoDeletePodWhenVolumeDetachedUnexpectedly: null
  autoSalvage: null
  backingImageCleanupWaitInterval: null
  backingImageRecoveryWaitInterval: null
  backupCompressionMethod: null
  backupConcurrentLimit: null
  backupTarget: null
  backupTargetCredentialSecret: null
  backupstorePollInterval: null
  concurrentAutomaticEngineUpgradePerNodeLimit: null
  concurrentReplicaRebuildPerNodeLimit: null
  concurrentVolumeBackupRestorePerNodeLimit: null
  createDefaultDiskLabeledNodes: null
  defaultDataLocality: null
  defaultDataPath: null
  defaultLonghornStaticStorageClass: null
  defaultReplicaCount: null
  deletingConfirmationFlag: null
  disableRevisionCounter: null
  disableSchedulingOnCordonedNode: null
  disableSnapshotPurge: null
  engineReplicaTimeout: null
  failedBackupTTL: null
  fastReplicaRebuildEnabled: null
  guaranteedInstanceManagerCPU: null
  kubernetesClusterAutoscalerEnabled: null
  logLevel: null
  nodeDownPodDeletionPolicy: null
  nodeDrainPolicy: null
  offlineReplicaRebuilding: null
  orphanAutoDeletion: null
  priorityClass: null
  recurringFailedJobsHistoryLimit: null
  recurringSuccessfulJobsHistoryLimit: null
  removeSnapshotsDuringFilesystemTrim: null
  replicaAutoBalance: null
  replicaFileSyncHttpClientTimeout: null
  replicaReplenishmentWaitInterval: null
  replicaSoftAntiAffinity: null
  replicaZoneSoftAntiAffinity: null
  restoreConcurrentLimit: null
  restoreVolumeRecurringJobs: null
  snapshotDataIntegrity: null
  snapshotDataIntegrityCronjob: null
  snapshotDataIntegrityImmediateCheckAfterSnapshotCreation: null
  storageMinimalAvailablePercentage: null
  storageNetwork: null
  storageOverProvisioningPercentage: null
  storageReservedPercentageForDefaultDisk: null
  supportBundleFailedHistoryLimit: null
  systemManagedComponentsNodeSelector: null
  systemManagedPodsImagePullPolicy: null
  taintToleration: null
  upgradeChecker: null
  v2DataEngine: null
enablePSP: false
global:
  cattle:
    windowsCluster:
      defaultSetting:
        systemManagedComponentsNodeSelector: <http://kubernetes.io/os:linux|kubernetes.io/os:linux>
        taintToleration: <http://cattle.io/os=linux:NoSchedule|cattle.io/os=linux:NoSchedule>
      enabled: false
      nodeSelector:
        <http://kubernetes.io/os|kubernetes.io/os>: linux
      tolerations:
        - effect: NoSchedule
          key: <http://cattle.io/os|cattle.io/os>
          operator: Equal
          value: linux
    systemProjectId: p-n5272
helmPreUpgradeCheckerJob:
  enabled: true
image:
  csi:
    attacher:
      repository: rancher/mirrored-longhornio-csi-attacher
      tag: v4.4.2
    livenessProbe:
      repository: rancher/mirrored-longhornio-livenessprobe
      tag: v2.12.0
    nodeDriverRegistrar:
      repository: rancher/mirrored-longhornio-csi-node-driver-registrar
      tag: v2.9.2
    provisioner:
      repository: rancher/mirrored-longhornio-csi-provisioner
      tag: v3.6.2
    resizer:
      repository: rancher/mirrored-longhornio-csi-resizer
      tag: v1.9.2
    snapshotter:
      repository: rancher/mirrored-longhornio-csi-snapshotter
      tag: v6.3.2
  longhorn:
    backingImageManager:
      repository: rancher/mirrored-longhornio-backing-image-manager
      tag: v1.5.5
    engine:
      repository: rancher/mirrored-longhornio-longhorn-engine
      tag: v1.5.5
    instanceManager:
      repository: rancher/mirrored-longhornio-longhorn-instance-manager
      tag: v1.5.5
    manager:
      repository: rancher/mirrored-longhornio-longhorn-manager
      tag: v1.5.5
    shareManager:
      repository: rancher/mirrored-longhornio-longhorn-share-manager
      tag: v1.5.5
    supportBundleKit:
      repository: rancher/mirrored-longhornio-support-bundle-kit
      tag: v0.0.37
    ui:
      repository: rancher/mirrored-longhornio-longhorn-ui
      tag: v1.5.5
  pullPolicy: IfNotPresent
  defaultImage: true
ingress:
  annotations: null
  enabled: false
  host: <http://sslip.io|sslip.io>
  ingressClassName: null
  path: /
  secrets: null
  secureBackends: false
  tls: false
  tlsSecret: longhorn.local-tls
longhornDriver:
  nodeSelector: {}
  priorityClass: null
  tolerations: []
longhornManager:
  log:
    format: plain
  nodeSelector: {}
  priorityClass: null
  serviceAnnotations: {}
  tolerations: []
longhornUI:
  nodeSelector: {}
  priorityClass: null
  replicas: 2
  tolerations: []
metrics:
  serviceMonitor:
    enabled: false
namespaceOverride: ''
networkPolicies:
  enabled: false
  type: k3s
persistence:
  backingImage:
    dataSourceParameters: null
    dataSourceType: null
    enable: false
    expectedChecksum: null
    name: null
  defaultClass: true
  defaultClassReplicaCount: 3
  defaultDataLocality: disabled
  defaultFsType: ext4
  defaultMkfsParams: ''
  defaultNodeSelector:
    enable: false
    selector: ''
  migratable: false
  nfsOptions: ''
  reclaimPolicy: Delete
  recurringJobSelector:
    enable: false
    jobList: []
  removeSnapshotsDuringFilesystemTrim: ignored
  recurringJobs:
    enable: false
privateRegistry:
  createSecret: null
  registryPasswd: null
  registrySecret: null
  registryUrl: null
  registryUser: null
service:
  manager:
    loadBalancerIP: ''
    loadBalancerSourceRanges: ''
    nodePort: ''
    type: ClusterIP
  ui:
    nodePort: null
    type: Rancher-Proxy
serviceAccount:
  annotations: {}
longhorn:
  default_setting: false

The longhorn-manager ds pods are all giving this error:

2024-05-15T11:17:29.573493567-05:00 E0515 16:17:29.573405       1 reflector.go:147] <http://github.com/longhorn/longhorn-manager/k8s/pkg/client/informers/externalversions/factory.go:117|github.com/longhorn/longhorn-manager/k8s/pkg/client/informers/externalversions/factory.go:117>: Failed to watch *v1beta2.BackupTarget: failed to list *v1beta2.BackupTarget: conversion webhook for <http://longhorn.io/v1beta1|longhorn.io/v1beta1>, Kind=BackupTarget failed: Post "<https://longhorn-conversion-webhook.longhorn-system.svc:9501/v1/webhook/conversion?timeout=30s>": no endpoints available for service "longhorn-conversion-webhook"
2024-05-15T11:18:23.403036747-05:00 W0515 16:18:23.402908       1 reflector.go:535] <http://github.com/longhorn/longhorn-manager/k8s/pkg/client/informers/externalversions/factory.go:117|github.com/longhorn/longhorn-manager/k8s/pkg/client/informers/externalversions/factory.go:117>: failed to list *v1beta2.BackupTarget: conversion webhook for <http://longhorn.io/v1beta1|longhorn.io/v1beta1>, Kind=BackupTarget failed: Post "<https://longhorn-conversion-webhook.longhorn-system.svc:9501/v1/webhook/conversion?timeout=30s>": no endpoints available for service "longhorn-conversion-webhook"

It looks like the services in question are set to select pods with the

app:longhorn-manager

label...which is just the ds pods in question that haven't started up.