metric unregistration on route removal, fixed multi-ips as visitor label detected from x headers

internal/metrics/metric.go

@@ -4,12 +4,12 @@ import "github.com/prometheus/client_golang/prometheus"
 
 type (
 	Counter struct {
-		collector prometheus.Counter
 		mv        *prometheus.CounterVec
+		collector prometheus.Counter
 	}
 	Gauge struct {
-		collector prometheus.Gauge
 		mv        *prometheus.GaugeVec
+		collector prometheus.Gauge
 	}
 	Labels interface {
 		toPromLabels() prometheus.Labels
@@ -52,8 +52,8 @@ func (c *Counter) Inc() {
 	c.collector.Inc()
 }
 
-func (c *Counter) With(l Labels) prometheus.Counter {
-	return c.mv.With(l.toPromLabels())
+func (c *Counter) With(l Labels) *Counter {
+	return &Counter{mv: c.mv, collector: c.mv.With(l.toPromLabels())}
 }
 
 func (g *Gauge) Collect(ch chan<- prometheus.Metric) {
@@ -68,6 +68,6 @@ func (g *Gauge) Set(v float64) {
 	g.collector.Set(v)
 }
 
-func (g *Gauge) With(l Labels) prometheus.Gauge {
-	return g.mv.With(l.toPromLabels())
+func (g *Gauge) With(l Labels) *Gauge {
+	return &Gauge{mv: g.mv, collector: g.mv.With(l.toPromLabels())}
 }

internal/metrics/metrics.go

@@ -7,16 +7,24 @@ import (
 	"github.com/yusing/go-proxy/internal/common"
 )
 
-type RouteMetrics struct {
-	HTTPReqTotal,
-	HTTP2xx3xx,
-	HTTP4xx,
-	HTTP5xx *Counter
-	HTTPReqElapsed *Gauge
-	HealthStatus   *Gauge
-}
+type (
+	RouteMetrics struct {
+		HTTPReqTotal,
+		HTTP2xx3xx,
+		HTTP4xx,
+		HTTP5xx *Counter
+		HTTPReqElapsed *Gauge
+	}
 
-var rm RouteMetrics
+	ServiceMetrics struct {
+		HealthStatus *Gauge
+	}
+)
+
+var (
+	rm RouteMetrics
+	sm ServiceMetrics
+)
 
 const (
 	routerNamespace     = "router"
@@ -29,10 +37,27 @@ func GetRouteMetrics() *RouteMetrics {
 	return &rm
 }
 
+func GetServiceMetrics() *ServiceMetrics {
+	return &sm
+}
+
+func (rm *RouteMetrics) UnregisterService(service string) {
+	lbls := &HTTPRouteMetricLabels{Service: service}
+	prometheus.Unregister(rm.HTTP2xx3xx.With(lbls))
+	prometheus.Unregister(rm.HTTP4xx.With(lbls))
+	prometheus.Unregister(rm.HTTP5xx.With(lbls))
+	prometheus.Unregister(rm.HTTPReqElapsed.With(lbls))
+}
+
 func init() {
 	if !common.PrometheusEnabled {
 		return
 	}
+	initRouteMetrics()
+	initServiceMetrics()
+}
+
+func initRouteMetrics() {
 	lbls := []string{"service", "method", "host", "visitor", "path"}
 	partitionsHelp := ", partitioned by " + strings.Join(lbls, ", ")
 	rm = RouteMetrics{
@@ -66,6 +91,11 @@ func init() {
 			Name:      "req_elapsed_ms",
 			Help:      "How long it took to process the request and respond a status code" + partitionsHelp,
 		}, lbls...),
+	}
+}
+
+func initServiceMetrics() {
+	sm = ServiceMetrics{
 		HealthStatus: NewGauge(prometheus.GaugeOpts{
 			Namespace: serviceNamespace,
 			Name:      "health_status",