diff --git a/cmd/iam.go b/cmd/iam.go index 0c7a34b3e..d015d2a47 100644 --- a/cmd/iam.go +++ b/cmd/iam.go @@ -29,6 +29,7 @@ import ( "sort" "strings" "sync" + "sync/atomic" "time" humanize "github.com/dustin/go-humanize" @@ -77,6 +78,12 @@ type IAMSys struct { // configLoaded will be closed and remain so after first load. configLoaded chan struct{} + + // metrics + LastRefreshTimeUnixNano uint64 + LastRefreshDurationMilliseconds uint64 + TotalRefreshSuccesses uint64 + TotalRefreshFailures uint64 } // IAMUserType represents a user type inside MinIO server @@ -184,10 +191,17 @@ func (sys *IAMSys) Initialized() bool { // Load - loads all credentials, policies and policy mappings. func (sys *IAMSys) Load(ctx context.Context) error { + loadStartTime := time.Now() err := sys.store.LoadIAMCache(ctx) if err != nil { + atomic.AddUint64(&sys.TotalRefreshFailures, 1) return err } + loadDuration := time.Since(loadStartTime) + + atomic.StoreUint64(&sys.LastRefreshDurationMilliseconds, uint64(loadDuration.Milliseconds())) + atomic.StoreUint64(&sys.LastRefreshTimeUnixNano, uint64(loadStartTime.Add(loadDuration).UnixNano())) + atomic.AddUint64(&sys.TotalRefreshSuccesses, 1) select { case <-sys.configLoaded: diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go index 80ee2c271..56cadd43b 100644 --- a/cmd/metrics-v2.go +++ b/cmd/metrics-v2.go @@ -62,6 +62,7 @@ func init() { getS3TTFBMetric(), getILMNodeMetrics(), getScannerNodeMetrics(), + getIAMNodeMetrics(), } allMetricsGroups := func() (allMetrics []*MetricsGroup) { @@ -121,6 +122,7 @@ const ( quotaSubsystem MetricSubsystem = "quota" ilmSubsystem MetricSubsystem = "ilm" scannerSubsystem MetricSubsystem = "scanner" + iamSubsystem MetricSubsystem = "iam" ) // MetricName are the individual names for the metric. @@ -1266,6 +1268,62 @@ func getScannerNodeMetrics() *MetricsGroup { return mg } +func getIAMNodeMetrics() *MetricsGroup { + mg := &MetricsGroup{} + mg.RegisterRead(func(_ context.Context) (metrics []Metric) { + lastSyncTime := atomic.LoadUint64(&globalIAMSys.LastRefreshTimeUnixNano) + var sinceLastSyncMillis uint64 + if lastSyncTime != 0 { + sinceLastSyncMillis = (uint64(time.Now().UnixNano()) - lastSyncTime) / uint64(time.Millisecond) + } + + metrics = []Metric{ + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "last_sync_duration_millis", + Help: "Last successful IAM data sync duration in milliseconds", + Type: gaugeMetric, + }, + Value: float64(atomic.LoadUint64(&globalIAMSys.LastRefreshDurationMilliseconds)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "since_last_sync_millis", + Help: "Time (in milliseconds) since last successful IAM data sync. This is set to 0 until the first sync after server start.", + Type: gaugeMetric, + }, + Value: float64(sinceLastSyncMillis), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "sync_successes", + Help: "Number of successful IAM data syncs since server start.", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshSuccesses)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "sync_failures", + Help: "Number of failed IAM data syncs since server start.", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshFailures)), + }, + } + return metrics + }) + return mg +} + func getMinioVersionMetrics() *MetricsGroup { mg := &MetricsGroup{} mg.RegisterRead(func(_ context.Context) (metrics []Metric) {