From ae3b369fe1eeec8123f10404efe246bf24253ed4 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Tue, 15 Mar 2022 17:45:51 -0700 Subject: [PATCH] logger webhook failure can overrun the queue_size (#14556) PR introduced in #13819 was incorrect and was not handling the situation where a buffer is full can cause incessant amount of logs that would keep the logger webhook overrun by the requests. To avoid this only log failures to console logger instead of all targets as it can cause self reference, leading to an infinite loop. --- internal/logger/logger.go | 5 ++++- internal/logger/targets.go | 11 ++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 5ef5244c7..9f80ace50 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -360,7 +360,10 @@ func logIf(ctx context.Context, err error, errKind ...interface{}) { // Iterate over all logger targets to send the log entry for _, t := range SystemTargets() { if err := t.Send(entry, entry.LogKind); err != nil { - LogAlwaysIf(context.Background(), fmt.Errorf("event(%v) was not sent to Logger target (%v): %v", entry, t, err), entry.LogKind) + if consoleTgt != nil { + entry.Trace.Message = fmt.Sprintf("event(%#v) was not sent to Logger target (%#v): %#v", entry, t, err) + consoleTgt.Send(entry, entry.LogKind) + } } } } diff --git a/internal/logger/targets.go b/internal/logger/targets.go index 4146a9053..9337bed5f 100644 --- a/internal/logger/targets.go +++ b/internal/logger/targets.go @@ -46,7 +46,11 @@ var ( // Must be immutable at all times. // Can be swapped to another while holding swapMu systemTargets = []Target{} - nTargets int32 // atomic count of len(targets) + + // This is always set represent /dev/console target + consoleTgt Target + + nTargets int32 // atomic count of len(targets) ) // SystemTargets returns active targets. @@ -90,6 +94,11 @@ func AddSystemTarget(t Target) error { return err } swapMu.Lock() + if consoleTgt == nil { + if t.Type() == types.TargetConsole { + consoleTgt = t + } + } updated := append(make([]Target, 0, len(systemTargets)+1), systemTargets...) updated = append(updated, t) systemTargets = updated