Add labels to expired backups failing garbage collection. (#4757)

* Add bsl related TTL gc errors to labelSelectors
* if backup label map is nil, make map
* clear label if not BSL error

Signed-off-by: Tiger Kaovilai <tkaovila@redhat.com>

pkg/controller/gc_controller.go

@@ -39,7 +39,11 @@ import (
 )
 
 const (
-	GCSyncPeriod = 60 * time.Minute
+	GCSyncPeriod             = 60 * time.Minute
+	garbageCollectionFailure = "velero.io/gc-failure"
+	gcFailureBSLNotFound     = "BSLNotFound"
+	gcFailureBSLCannotGet    = "BSLCannotGet"
+	gcFailureBSLReadOnly     = "BSLReadOnly"
 )
 
 // gcController creates DeleteBackupRequests for expired backups.
@@ -134,6 +138,10 @@ func (c *gcController) processQueueItem(key string) error {
 
 	log.Info("Backup has expired")
 
+	if backup.Labels == nil {
+		backup.Labels = make(map[string]string)
+	}
+
 	loc := &velerov1api.BackupStorageLocation{}
 	if err := c.kbClient.Get(context.Background(), client.ObjectKey{
 		Namespace: ns,
@@ -141,15 +149,31 @@ func (c *gcController) processQueueItem(key string) error {
 	}, loc); err != nil {
 		if apierrors.IsNotFound(err) {
 			log.Warnf("Backup cannot be garbage-collected because backup storage location %s does not exist", backup.Spec.StorageLocation)
+			backup.Labels[garbageCollectionFailure] = gcFailureBSLNotFound
+		} else {
+			backup.Labels[garbageCollectionFailure] = gcFailureBSLCannotGet
+		}
+		if err := c.kbClient.Update(context.Background(), backup); err != nil {
+			log.WithError(err).Error("error updating backup labels")
 		}
 		return errors.Wrap(err, "error getting backup storage location")
 	}
 
 	if loc.Spec.AccessMode == velerov1api.BackupStorageLocationAccessModeReadOnly {
 		log.Infof("Backup cannot be garbage-collected because backup storage location %s is currently in read-only mode", loc.Name)
+		backup.Labels[garbageCollectionFailure] = gcFailureBSLReadOnly
+		if err := c.kbClient.Update(context.Background(), backup); err != nil {
+			log.WithError(err).Error("error updating backup labels")
+		}
 		return nil
 	}
 
+	// remove gc fail error label after this point
+	delete(backup.Labels, garbageCollectionFailure)
+	if err := c.kbClient.Update(context.Background(), backup); err != nil {
+		log.WithError(err).Error("error updating backup labels")
+	}
+
 	selector := labels.SelectorFromSet(labels.Set(map[string]string{
 		velerov1api.BackupNameLabel: label.GetValidName(backup.Name),
 		velerov1api.BackupUIDLabel:  string(backup.UID),