From 450fa72fbb765f344af88bc8bda008b5d89d5861 Mon Sep 17 00:00:00 2001
From: Alex Lemaresquier <alex+github@lemaresquier.org>
Date: Fri, 20 Jul 2018 15:03:44 +0200
Subject: [PATCH] Initialize schedule Prometheus metrics to have them created
 beforehand (see
 https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics)
 Signed-off-by: Alex Lemaresquier <alex+github@lemaresquier.org>

---
 pkg/cmd/server/server.go                   |  1 +
 pkg/controller/schedule_controller.go      |  8 ++++++++
 pkg/controller/schedule_controller_test.go |  2 ++
 pkg/metrics/metrics.go                     | 12 ++++++++++++
 4 files changed, 23 insertions(+)

diff --git a/pkg/cmd/server/server.go b/pkg/cmd/server/server.go
index 21a4ef909..7b6f53253 100644
--- a/pkg/cmd/server/server.go
+++ b/pkg/cmd/server/server.go
@@ -656,6 +656,7 @@ func (s *server) runControllers(config *api.Config) error {
 			s.sharedInformerFactory.Ark().V1().Schedules(),
 			config.ScheduleSyncPeriod.Duration,
 			s.logger,
+			s.metrics,
 		)
 		wg.Add(1)
 		go func() {
diff --git a/pkg/controller/schedule_controller.go b/pkg/controller/schedule_controller.go
index 0ce260281..5f01a5005 100644
--- a/pkg/controller/schedule_controller.go
+++ b/pkg/controller/schedule_controller.go
@@ -41,6 +41,7 @@ import (
 	arkv1client "github.com/heptio/ark/pkg/generated/clientset/versioned/typed/ark/v1"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions/ark/v1"
 	listers "github.com/heptio/ark/pkg/generated/listers/ark/v1"
+	"github.com/heptio/ark/pkg/metrics"
 	kubeutil "github.com/heptio/ark/pkg/util/kube"
 )
 
@@ -55,6 +56,7 @@ type scheduleController struct {
 	syncPeriod            time.Duration
 	clock                 clock.Clock
 	logger                logrus.FieldLogger
+	metrics               *metrics.ServerMetrics
 }
 
 func NewScheduleController(
@@ -64,6 +66,7 @@ func NewScheduleController(
 	schedulesInformer informers.ScheduleInformer,
 	syncPeriod time.Duration,
 	logger logrus.FieldLogger,
+	metrics *metrics.ServerMetrics,
 ) *scheduleController {
 	if syncPeriod < time.Minute {
 		logger.WithField("syncPeriod", syncPeriod).Info("Provided schedule sync period is too short. Setting to 1 minute")
@@ -80,6 +83,7 @@ func NewScheduleController(
 		syncPeriod: syncPeriod,
 		clock:      clock.RealClock{},
 		logger:     logger,
+		metrics:    metrics,
 	}
 
 	c.syncHandler = c.processSchedule
@@ -106,6 +110,10 @@ func NewScheduleController(
 					return
 				}
 				c.queue.Add(key)
+				scheduleName := schedule.GetName()
+				c.logger.Info("Creating schedule ", scheduleName)
+				//Init Prometheus metrics to 0 to have them flowing up
+				metrics.InitSchedule(scheduleName)
 			},
 		},
 	)
diff --git a/pkg/controller/schedule_controller_test.go b/pkg/controller/schedule_controller_test.go
index 07917661d..c35130a4c 100644
--- a/pkg/controller/schedule_controller_test.go
+++ b/pkg/controller/schedule_controller_test.go
@@ -34,6 +34,7 @@ import (
 	api "github.com/heptio/ark/pkg/apis/ark/v1"
 	"github.com/heptio/ark/pkg/generated/clientset/versioned/fake"
 	informers "github.com/heptio/ark/pkg/generated/informers/externalversions"
+	"github.com/heptio/ark/pkg/metrics"
 	"github.com/heptio/ark/pkg/util/collections"
 	arktest "github.com/heptio/ark/pkg/util/test"
 )
@@ -129,6 +130,7 @@ func TestProcessSchedule(t *testing.T) {
 				sharedInformers.Ark().V1().Schedules(),
 				time.Duration(0),
 				logger,
+				metrics.NewServerMetrics(),
 			)
 
 			var (
diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go
index c63725c18..75115c564 100644
--- a/pkg/metrics/metrics.go
+++ b/pkg/metrics/metrics.go
@@ -106,6 +106,18 @@ func (m *ServerMetrics) RegisterAllMetrics() {
 	}
 }
 
+func (m *ServerMetrics) InitSchedule(scheduleName string) {
+	if c, ok := m.metrics[backupAttemptCount].(*prometheus.CounterVec); ok {
+		c.WithLabelValues(scheduleName).Set(0)
+	}
+	if c, ok := m.metrics[backupSuccessCount].(*prometheus.CounterVec); ok {
+		c.WithLabelValues(scheduleName).Set(0)
+	}
+	if c, ok := m.metrics[backupFailureCount].(*prometheus.CounterVec); ok {
+		c.WithLabelValues(scheduleName).Set(0)
+	}
+}
+
 // SetBackupTarballSizeBytesGauge records the size, in bytes, of a backup tarball.
 func (m *ServerMetrics) SetBackupTarballSizeBytesGauge(backupSchedule string, size int64) {
 	if g, ok := m.metrics[backupTarballSizeBytesGauge].(*prometheus.GaugeVec); ok {