feat: notifications retry mechanism and improved error formatting

internal/notif/dispatcher.go

@@ -1,18 +1,21 @@
 package notif
 
 import (
+	"time"
+
 	"github.com/rs/zerolog"
 	"github.com/yusing/go-proxy/internal/gperr"
-	"github.com/yusing/go-proxy/internal/logging"
 	"github.com/yusing/go-proxy/internal/task"
 	F "github.com/yusing/go-proxy/internal/utils/functional"
 )
 
 type (
 	Dispatcher struct {
-		task      *task.Task
-		logCh     chan *LogMessage
-		providers F.Set[Provider]
+		task        *task.Task
+		providers   F.Set[Provider]
+		logCh       chan *LogMessage
+		retryCh     chan *RetryMessage
+		retryTicker *time.Ticker
 	}
 	LogMessage struct {
 		Level zerolog.Level
@@ -20,17 +23,33 @@ type (
 		Body  LogBody
 		Color Color
 	}
+	RetryMessage struct {
+		Message  *LogMessage
+		Trials   int
+		Provider Provider
+	}
 )
 
 var dispatcher *Dispatcher
 
-const dispatchErr = "notification dispatch error"
+const retryInterval = 5 * time.Second
+
+var maxRetries = map[zerolog.Level]int{
+	zerolog.DebugLevel: 1,
+	zerolog.InfoLevel:  1,
+	zerolog.WarnLevel:  3,
+	zerolog.ErrorLevel: 5,
+	zerolog.FatalLevel: 10,
+	zerolog.PanicLevel: 10,
+}
 
 func StartNotifDispatcher(parent task.Parent) *Dispatcher {
 	dispatcher = &Dispatcher{
-		task:      parent.Subtask("notification"),
-		logCh:     make(chan *LogMessage),
-		providers: F.NewSet[Provider](),
+		task:        parent.Subtask("notification"),
+		providers:   F.NewSet[Provider](),
+		logCh:       make(chan *LogMessage),
+		retryCh:     make(chan *RetryMessage, 100),
+		retryTicker: time.NewTicker(retryInterval),
 	}
 	go dispatcher.start()
 	return dispatcher
@@ -48,10 +67,6 @@ func Notify(msg *LogMessage) {
 	}
 }
 
-func (f *FieldsBody) Add(name, value string) {
-	*f = append(*f, LogField{Name: name, Value: value})
-}
-
 func (disp *Dispatcher) RegisterProvider(cfg *NotificationConfig) {
 	disp.providers.Add(cfg.Provider)
 }
@@ -61,6 +76,7 @@ func (disp *Dispatcher) start() {
 		dispatcher = nil
 		disp.providers.Clear()
 		close(disp.logCh)
+		close(disp.retryCh)
 		disp.task.Finish(nil)
 	}()
 
@@ -73,6 +89,23 @@ func (disp *Dispatcher) start() {
 				return
 			}
 			go disp.dispatch(msg)
+		case <-disp.retryTicker.C:
+			if len(disp.retryCh) == 0 {
+				continue
+			}
+			var msgs []*RetryMessage
+			done := false
+			for !done {
+				select {
+				case msg := <-disp.retryCh:
+					msgs = append(msgs, msg)
+				default:
+					done = true
+				}
+			}
+			if err := disp.retry(msgs); err != nil {
+				gperr.LogError("notification retry failed", err)
+			}
 		}
 	}
 }
@@ -81,15 +114,34 @@ func (disp *Dispatcher) dispatch(msg *LogMessage) {
 	task := disp.task.Subtask("dispatcher")
 	defer task.Finish("notif dispatched")
 
-	errs := gperr.NewBuilderWithConcurrency(dispatchErr)
 	disp.providers.RangeAllParallel(func(p Provider) {
-		if err := notifyProvider(task.Context(), p, msg); err != nil {
-			errs.Add(gperr.PrependSubject(p.GetName(), err))
+		if err := msg.notify(task.Context(), p); err != nil {
+			disp.retryCh <- &RetryMessage{
+				Message:  msg,
+				Trials:   0,
+				Provider: p,
+			}
 		}
 	})
-	if errs.HasError() {
-		gperr.LogError(errs.About(), errs.Error())
-	} else {
-		logging.Debug().Str("title", msg.Title).Msgf("dispatched notif")
-	}
+}
+
+func (disp *Dispatcher) retry(messages []*RetryMessage) error {
+	task := disp.task.Subtask("retry")
+	defer task.Finish("notif retried")
+
+	errs := gperr.NewBuilder("notification failure")
+	for _, msg := range messages {
+		err := msg.Message.notify(task.Context(), msg.Provider)
+		if err == nil {
+			continue
+		}
+		if msg.Trials > maxRetries[msg.Message.Level] {
+			errs.Addf("notification provider %s failed after %d trials", msg.Provider.GetName(), msg.Trials)
+			errs.Add(err)
+			continue
+		}
+		msg.Trials++
+		disp.retryCh <- msg
+	}
+	return errs.Error()
 }