mirrors/object-browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implemented Log Search API & Prometheus functionality (#549)

Browse Source 
Implemented Log Search API & Prometheus functionality in console, also fixed minor issues in all the platform

Co-authored-by: Benjamin Perez <benjamin@bexsoft.net>
This commit is contained in:
Alex
2021-01-13 14:08:32 -06:00
committed by GitHub
parent f3bcfc327d
commit 1c109769df
78 changed files with 6753 additions and 714 deletions
Download Patch File Download Diff File Expand all files Collapse all files

624
restapi/admin_info.go
View File

@@ -18,6 +18,14 @@ package restapi
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"net/url"
"regexp"
"strings"
"time"
"github.com/go-openapi/runtime/middleware"
@@ -29,7 +37,7 @@ import (
func registerAdminInfoHandlers(api *operations.ConsoleAPI) {
// return usage stats
api.AdminAPIAdminInfoHandler = admin_api.AdminInfoHandlerFunc(func(params admin_api.AdminInfoParams, session *models.Principal) middleware.Responder {
infoResp, err := getAdminInfoResponse(session)
infoResp, err := getAdminInfoResponse(session, params)
if err != nil {
return admin_api.NewAdminInfoDefault(int(err.Code)).WithPayload(err)
}
@@ -69,27 +77,605 @@ func getAdminInfo(ctx context.Context, client MinioAdmin) (*usageInfo, error) {
}, nil
}
type Target struct {
Expr string
Interval string
LegendFormat string
}
type ReduceOptions struct {
Calcs []string
}
type MetricOptions struct {
ReduceOptions ReduceOptions
}
type Metric struct {
Title string
Type string
Options MetricOptions
Targets []Target
}
type WidgetLabel struct {
Name string
}
var labels = []WidgetLabel{
{Name: "instance"},
{Name: "disk"},
}
var widgets = []Metric{
{
Title: "Uptime",
Type: "stat",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"mean",
},
},
},
Targets: []Target{
{
Expr: "time() - max(process_start_time_seconds)",
LegendFormat: "{{instance}}",
},
},
},
{
Title: "Total Online disks",
Type: "stat",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"mean",
},
},
},
Targets: []Target{
{
Expr: "sum(minio_disks_total)",
LegendFormat: "Total online disks in MinIO Cluster",
},
},
},
{
Title: "Total Data",
Type: "gauge",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"lastNotNull",
},
},
},
Targets: []Target{
{
Expr: "topk(1, sum(bucket_usage_size) by (instance))",
LegendFormat: "",
},
},
},
{
Title: "Data Growth",
Type: "graph",
Targets: []Target{
{
Expr: "topk(1, sum(bucket_usage_size) by (instance))",
LegendFormat: "Total Storage Used",
},
},
},
{
Title: "Object size distribution",
Type: "bargauge",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"mean",
},
},
},
Targets: []Target{
{
Expr: "max by (object_size) (bucket_objects_histogram)",
LegendFormat: "{{object_size}}",
},
},
},
{
Title: "Total Offline disks",
Type: "singlestat",
Targets: []Target{
{
Expr: "sum(minio_disks_offline)",
LegendFormat: "Total offline disks in MinIO Cluster",
},
},
},
{
Title: "Total Online Servers",
Type: "stat",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"mean",
},
},
},
Targets: []Target{
{
Expr: "count by (instances) (minio_version_info)",
LegendFormat: "",
},
},
},
{
Title: "Total S3 Traffic Inbound",
Type: "stat",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"mean",
},
},
},
Targets: []Target{
{
Expr: "sum without (instance) (s3_rx_bytes_total)",
LegendFormat: "",
},
},
},
{
Title: "Number of Buckets",
Type: "stat",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"lastNotNull",
},
},
},
Targets: []Target{
{
Expr: "count(count by (bucket) (bucket_objects_count))",
LegendFormat: "",
},
},
},
{
Title: "S3 API Request & Error Rate",
Type: "graph",
Targets: []Target{
{
Expr: "sum without (instance,api)(rate(s3_requests_total[10m]))",
LegendFormat: "S3 Requests",
},
{
Expr: "sum without (instance,api)(rate(s3_errors_total[10m]))",
LegendFormat: "S3 Errors",
},
},
},
{
Title: "Total Open FDs",
Type: "stat",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"mean",
},
},
},
Targets: []Target{
{
Expr: "sum without (instance)(process_open_fds)",
LegendFormat: "",
},
},
},
{
Title: "Total S3 Traffic Outbound",
Type: "stat",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"mean",
},
},
},
Targets: []Target{
{
Expr: "sum without (instance)(s3_tx_bytes_total)",
LegendFormat: "",
},
},
},
{
Title: "Number of Objects",
Type: "stat",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"lastNotNull",
},
},
},
Targets: []Target{
{
Expr: "topk(1, sum(bucket_objects_count) by (instance))",
LegendFormat: "",
},
},
},
{
Title: "Total Goroutines",
Type: "stat",
Options: MetricOptions{
ReduceOptions: ReduceOptions{
Calcs: []string{
"mean",
},
},
},
Targets: []Target{
{
Expr: "sum without (instance) (go_goroutines)",
LegendFormat: "",
},
},
},
{
Title: "S3 API Data Transfer",
Type: "graph",
Targets: []Target{
{
Expr: "sum without (instance,api)(rate(s3_tx_bytes_total[5m]))",
LegendFormat: "S3 Data Sent",
},
{
Expr: "sum without (instance,api)(rate(s3_rx_bytes_total[5m]))",
LegendFormat: "S3 Data Received",
},
},
},
{
Title: "Total S3 API Data Transfer",
Type: "graph",
Targets: []Target{
{
Expr: "sum without (instance) (s3_rx_bytes_total)",
LegendFormat: "S3 Bytes Received {{instance}}",
},
{
Expr: "sum without (instance) (s3_tx_bytes_total)",
LegendFormat: "S3 Bytes Sent {{instance}}",
},
},
},
{
Title: "Active S3 Requests",
Type: "graph",
Targets: []Target{
{
Expr: "s3_requests_current{instance=~\"$instance\"}",
LegendFormat: "Instance {{instance}} function {{api}}",
},
},
},
{
Title: "Internode Data Transfer",
Type: "graph",
Targets: []Target{
{
Expr: "internode_rx_bytes_total{instance=~\"$instance\"}",
LegendFormat: "Internode Bytes Received {{instance}}",
},
{
Expr: "internode_tx_bytes_total{instance=~\"$instance\"}",
LegendFormat: "Internode Bytes Sent {{instance}}",
},
},
},
{
Title: "Online Disks",
Type: "graph",
Targets: []Target{
{
Expr: "minio_disks_total{instance=~\"$instance\"} - minio_disks_offline{instance=~\"$instance\"}",
LegendFormat: "Online Disks {{instance}}",
},
},
},
{
Title: "Disk Usage",
Type: "graph",
Targets: []Target{
{
Expr: "disk_storage_used{disk=~\"$disk\",instance=~\"$instance\"}",
LegendFormat: "Used Capacity {{instance}} {{disk}}",
},
},
},
}
type Widget struct {
Title string
Type string
}
type DataResult struct {
Metric map[string]string `json:"metric"`
Values []interface{} `json:"values"`
}
type PromRespData struct {
ResultType string `json:"resultType"`
Result []DataResult `json:"result"`
}
type PromResp struct {
Status string `json:"status"`
Data PromRespData `json:"data"`
}
type LabelResponse struct {
Status string `json:"status"`
Data []string `json:"data"`
}
type LabelResults struct {
Label string
Response LabelResponse
}
// getAdminInfoResponse returns the response containing total buckets, objects and usage.
func getAdminInfoResponse(session *models.Principal) (*models.AdminInfoResponse, *models.Error) {
mAdmin, err := newMAdminClient(session)
if err != nil {
return nil, prepareError(err)
func getAdminInfoResponse(session *models.Principal, params admin_api.AdminInfoParams) (*models.AdminInfoResponse, *models.Error) {
prometheusURL := getPrometheusURL()
if prometheusURL == "" {
mAdmin, err := newMAdminClient(session)
if err != nil {
return nil, prepareError(err)
}
// create a minioClient interface implementation
// defining the client to be used
adminClient := adminClient{client: mAdmin}
// 20 seconds timeout
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
// serialize output
usage, err := getAdminInfo(ctx, adminClient)
if err != nil {
return nil, prepareError(err)
}
sessionResp := &models.AdminInfoResponse{
Buckets: usage.Buckets,
Objects: usage.Objects,
Usage: usage.Usage,
}
return sessionResp, nil
}
// create a minioClient interface implementation
// defining the client to be used
adminClient := adminClient{client: mAdmin}
// 20 seconds timeout
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
// serialize output
usage, err := getAdminInfo(ctx, adminClient)
if err != nil {
return nil, prepareError(err)
labelResultsCh := make(chan LabelResults)
for _, lbl := range labels {
go func(lbl WidgetLabel) {
endpoint := fmt.Sprintf("%s/api/v1/label/%s/values", prometheusURL, lbl.Name)
resp, err := http.Get(endpoint)
if err != nil {
log.Println(err)
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Println(err)
}
}()
if resp.StatusCode != 200 {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println(err)
return
}
log.Println(endpoint)
log.Println(resp.StatusCode)
log.Println(string(body))
return
}
var response LabelResponse
jd := json.NewDecoder(resp.Body)
if err = jd.Decode(&response); err != nil {
log.Println(err)
return
}
labelResultsCh <- LabelResults{Label: lbl.Name, Response: response}
}(lbl)
}
sessionResp := &models.AdminInfoResponse{
Buckets: usage.Buckets,
Objects: usage.Objects,
Usage: usage.Usage,
labelMap := make(map[string][]string)
// wait for as many goroutines that come back in less than 1 second
LabelsWaitLoop:
for {
select {
case <-time.After(1 * time.Second):
break LabelsWaitLoop
case res := <-labelResultsCh:
labelMap[res.Label] = res.Response.Data
if len(labelMap) >= len(labels) {
break LabelsWaitLoop
}
}
}
// launch a goroutines per widget
results := make(chan models.Widget)
for _, m := range widgets {
go func(m Metric, params admin_api.AdminInfoParams) {
targetResults := make(chan *models.ResultTarget)
// for each target we will launch another goroutine to fetch the values
for _, target := range m.Targets {
go func(target Target, params admin_api.AdminInfoParams) {
apiType := "query_range"
now := time.Now()
extraParamters := fmt.Sprintf("&start=%d&end=%d&step=%d", now.Add(-15*time.Minute).Unix(), now.Unix(), *params.Step)
if params.Start != nil && params.End != nil {
extraParamters = fmt.Sprintf("&start=%d&end=%d&step=%d", *params.Start, *params.End, *params.Step)
}
queryExpr := target.Expr
if strings.Contains(queryExpr, "$") {
var re = regexp.MustCompile(`\$([a-z]+)`)
for _, match := range re.FindAllStringSubmatch(queryExpr, -1) {
if val, ok := labelMap[match[1]]; ok {
queryExpr = strings.ReplaceAll(queryExpr, "$"+match[1], fmt.Sprintf("(%s)", strings.Join(val, "|")))
}
}
}
endpoint := fmt.Sprintf("%s/api/v1/%s?query=%s%s", getPrometheusURL(), apiType, url.QueryEscape(queryExpr), extraParamters)
resp, err := http.Get(endpoint)
if err != nil {
log.Println(err)
return
}
defer func() {
if err := resp.Body.Close(); err != nil {
log.Println(err)
}
}()
if resp.StatusCode != 200 {
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Println(err)
return
}
log.Println(endpoint)
log.Println(resp.StatusCode)
log.Println(string(body))
return
}
var response PromResp
jd := json.NewDecoder(resp.Body)
if err = jd.Decode(&response); err != nil {
log.Println(err)
return
}
//body, _ := ioutil.ReadAll(resp.Body)
//err = json.Unmarshal(body, &response)
//if err != nil {
// log.Println(err)
//}
targetResult := models.ResultTarget{
LegendFormat: target.LegendFormat,
ResultType: response.Data.ResultType,
}
for _, r := range response.Data.Result {
targetResult.Result = append(targetResult.Result, &models.WidgetResult{
Metric: r.Metric,
Values: r.Values,
})
}
//xx, err := json.Marshal(response)
//if err != nil {
// log.Println(err)
//}
//log.Println("----", m.Title)
//log.Println(string(body))
//log.Println(string(xx))
//log.Println("=====")
targetResults <- &targetResult
}(target, params)
}
wdgtResult := models.Widget{
Title: m.Title,
Type: m.Type,
}
if len(m.Options.ReduceOptions.Calcs) > 0 {
wdgtResult.Options = &models.WidgetOptions{
ReduceOptions: &models.WidgetOptionsReduceOptions{
Calcs: m.Options.ReduceOptions.Calcs,
},
}
}
// count how many targets we have received
targetsReceived := 0
for res := range targetResults {
wdgtResult.Targets = append(wdgtResult.Targets, res)
targetsReceived++
// upon receiving the total number of targets needed, we can close the channel to not lock the goroutine
if targetsReceived >= len(m.Targets) {
close(targetResults)
}
}
results <- wdgtResult
}(m, params)
}
// count the number of widgets that have completed calculating
totalWidgets := 0
sessionResp := &models.AdminInfoResponse{}
var wdgts []*models.Widget
// wait for as many goroutines that come back in less than 1 second
WaitLoop:
for {
select {
case <-time.After(1 * time.Second):
break WaitLoop
case res := <-results:
wdgts = append(wdgts, &res)
totalWidgets++
if totalWidgets >= len(widgets) {
break WaitLoop
}
}
}
sessionResp.Widgets = wdgts
return sessionResp, nil
}

81
restapi/admin_tenants.go
View File

@@ -30,6 +30,7 @@ import (
"strings"
"time"
"gopkg.in/yaml.v2"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/apimachinery/pkg/types"
@@ -310,7 +311,7 @@ func getTenantInfo(tenant *operator.Tenant) *models.Tenant {
}
var deletion string
if tenant.ObjectMeta.DeletionTimestamp != nil {
deletion = tenant.ObjectMeta.DeletionTimestamp.String()
deletion = tenant.ObjectMeta.DeletionTimestamp.Format(time.RFC3339)
}
if tenant.HasConsoleEnabled() {
@@ -318,7 +319,7 @@ func getTenantInfo(tenant *operator.Tenant) *models.Tenant {
}
return &models.Tenant{
CreationDate: tenant.ObjectMeta.CreationTimestamp.String(),
CreationDate: tenant.ObjectMeta.CreationTimestamp.Format(time.RFC3339),
DeletionDate: deletion,
Name: tenant.Name,
TotalSize: totalSize,
@@ -351,14 +352,16 @@ func getTenantInfoResponse(session *models.Principal, params admin_api.TenantInf
info := getTenantInfo(minTenant)
clientSet, err := cluster.K8sClient(session.STSSessionToken)
if err != nil {
return nil, prepareError(err)
}
k8sClient := k8sClient{
client: clientSet,
}
if minTenant.Spec.Console != nil {
clientSet, err := cluster.K8sClient(session.STSSessionToken)
k8sClient := k8sClient{
client: clientSet,
}
if err != nil {
return nil, prepareError(err)
}
// obtain current subnet license for tenant (if exists)
license, _ := getSubscriptionLicense(context.Background(), &k8sClient, params.Namespace, minTenant.Spec.Console.ConsoleSecret.Name)
if license != "" {
@@ -373,6 +376,40 @@ func getTenantInfoResponse(session *models.Principal, params admin_api.TenantInf
}
}
// get tenant service
minTenant.EnsureDefaults()
//minio service
minSvc, err := k8sClient.getService(ctx, minTenant.Namespace, minTenant.MinIOCIServiceName(), metav1.GetOptions{})
if err != nil {
return nil, prepareError(err)
}
//console service
conSvc, err := k8sClient.getService(ctx, minTenant.Namespace, minTenant.ConsoleCIServiceName(), metav1.GetOptions{})
if err != nil {
return nil, prepareError(err)
}
schema := "http"
consolePort := ":9090"
if minTenant.TLS() {
schema = "https"
consolePort = ":9443"
}
var minioEndpoint string
var consoleEndpoint string
if len(minSvc.Status.LoadBalancer.Ingress) > 0 {
minioEndpoint = fmt.Sprintf("%s://%s", schema, minSvc.Status.LoadBalancer.Ingress[0].IP)
}
if len(conSvc.Status.LoadBalancer.Ingress) > 0 {
consoleEndpoint = fmt.Sprintf("%s://%s%s", schema, conSvc.Status.LoadBalancer.Ingress[0].IP, consolePort)
}
if minioEndpoint != "" || consoleEndpoint != "" {
info.Endpoints = &models.TenantEndpoints{
Console: consoleEndpoint,
Minio: minioEndpoint,
}
}
return info, nil
}
@@ -407,11 +444,11 @@ func listTenants(ctx context.Context, operatorClient OperatorClientI, namespace
var deletion string
if tenant.ObjectMeta.DeletionTimestamp != nil {
deletion = tenant.ObjectMeta.DeletionTimestamp.String()
deletion = tenant.ObjectMeta.DeletionTimestamp.Format(time.RFC3339)
}
tenants = append(tenants, &models.TenantList{
CreationDate: tenant.ObjectMeta.CreationTimestamp.String(),
CreationDate: tenant.ObjectMeta.CreationTimestamp.Format(time.RFC3339),
DeletionDate: deletion,
Name: tenant.ObjectMeta.Name,
PoolCount: int64(len(tenant.Spec.Pools)),
@@ -702,7 +739,7 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
minInst.Spec.Console = &operator.ConsoleConfiguration{
Replicas: 1,
Image: ConsoleImageVersion,
Image: getConsoleImage(),
ConsoleSecret: &corev1.LocalObjectReference{Name: consoleSecretName},
Resources: corev1.ResourceRequirements{
Requests: map[corev1.ResourceName]resource.Quantity{
@@ -808,6 +845,26 @@ func getTenantCreatedResponse(session *models.Principal, params admin_api.Create
if tenantReq.ConsoleImage != "" {
minInst.Spec.Console.Image = tenantReq.ConsoleImage
}
// default activate lgo search and prometheus
minInst.Spec.Log = &operator.LogConfig{
Image: "miniodev/logsearch:v4.0.0",
Audit: &operator.AuditConfig{DiskCapacityGB: swag.Int(10)},
}
minInst.Spec.Prometheus = &operator.PrometheusConfig{
DiskCapacityDB: swag.Int(5),
}
// expose services
if tenantReq.ExposeMinio || tenantReq.ExposeConsole {
minInst.Spec.ExposeServices = &operator.ExposeServices{
MinIO: tenantReq.ExposeMinio,
Console: tenantReq.ExposeConsole,
}
log.Println("happened")
}
yo, _ := yaml.Marshal(minInst)
log.Println(string(yo))
opClient, err := cluster.OperatorClient(session.STSSessionToken)
if err != nil {

11
restapi/admin_tenants_test.go
View File

@@ -26,6 +26,7 @@ import (
"net/http"
"reflect"
"testing"
"time"
"github.com/go-openapi/swag"
"github.com/minio/console/cluster"
@@ -325,7 +326,7 @@ func Test_TenantInfo(t *testing.T) {
},
},
want: &models.Tenant{
CreationDate: testTimeStamp.String(),
CreationDate: testTimeStamp.Format(time.RFC3339),
Name: "tenant1",
TotalSize: int64(8388608),
CurrentState: "ready",
@@ -389,8 +390,8 @@ func Test_TenantInfo(t *testing.T) {
},
},
want: &models.Tenant{
CreationDate: testTimeStamp.String(),
DeletionDate: testTimeStamp.String(),
CreationDate: testTimeStamp.Format(time.RFC3339),
DeletionDate: testTimeStamp.Format(time.RFC3339),
Name: "tenant1",
TotalSize: int64(8388608),
CurrentState: "ready",
@@ -435,7 +436,7 @@ func Test_TenantInfo(t *testing.T) {
},
},
want: &models.Tenant{
CreationDate: testTimeStamp.String(),
CreationDate: testTimeStamp.Format(time.RFC3339),
Name: "tenant1",
CurrentState: "ready",
Namespace: "minio-ns",
@@ -470,7 +471,7 @@ func Test_TenantInfo(t *testing.T) {
},
},
want: &models.Tenant{
CreationDate: testTimeStamp.String(),
CreationDate: testTimeStamp.Format(time.RFC3339),
Name: "tenant1",
CurrentState: "ready",
Namespace: "minio-ns",

3
restapi/admin_trace.go
View File

@@ -23,6 +23,7 @@ import (
"log"
"net/http"
"strings"
"time"
"github.com/gorilla/websocket"
"github.com/minio/minio/pkg/madmin"
@@ -92,7 +93,7 @@ func shortTrace(info *madmin.ServiceTraceInfo) shortTraceMsg {
t := info.Trace
s := shortTraceMsg{}
s.Time = t.ReqInfo.Time.String()
s.Time = t.ReqInfo.Time.Format(time.RFC3339)
s.Path = t.ReqInfo.Path
s.Query = t.ReqInfo.RawQuery
s.FuncName = t.FuncName

54
restapi/config.go
View File

@@ -19,8 +19,10 @@ package restapi
import (
"crypto/x509"
"fmt"
"io/ioutil"
"strconv"
"strings"
"sync"
"time"
"github.com/minio/minio/pkg/certs"
@@ -44,6 +46,13 @@ var TLSRedirect = "off"
var SessionDuration = 45 * time.Minute
var logSearchAPI string
var logSearchURL string
var prometheusURL string
var consoleImage string
var once sync.Once
func getMinIOServer() string {
return strings.TrimSpace(env.Get(ConsoleMinIOServer, "http://localhost:9000"))
}
@@ -220,11 +229,39 @@ func getSecureExpectCTHeader() string {
return env.Get(ConsoleSecureExpectCTHeader, "")
}
func getLogSearchAPIToken() string {
once.Do(func() {
initVars()
})
return logSearchAPI
}
func getLogSearchURL() string {
once.Do(func() {
initVars()
})
return logSearchURL
}
func getPrometheusURL() string {
once.Do(func() {
initVars()
})
return prometheusURL
}
// GetSubnetLicense returns the current subnet jwt license
func GetSubnetLicense() string {
return env.Get(ConsoleSubnetLicense, "")
}
func initVars() {
logSearchAPI = env.Get(LogSearchQueryAuthToken, "")
logSearchURL = env.Get(LogSearchURL, "http://localhost:8080")
prometheusURL = env.Get(PrometheusURL, "")
consoleImage = env.Get(ConsoleOperatorConsoleImage, ConsoleImageDefaultVersion)
}
var (
// GlobalRootCAs is CA root certificates, a nil value means system certs pool will be used
GlobalRootCAs *x509.CertPool
@@ -233,3 +270,20 @@ var (
// GlobalTLSCertsManager custom TLS Manager for SNI support
GlobalTLSCertsManager *certs.Manager
)
// getK8sSAToken assumes the plugin is running inside a k8s pod and extract the current service account from the
// /var/run/secrets/kubernetes.io/serviceaccount/token file
func getK8sSAToken() string {
dat, err := ioutil.ReadFile("/var/run/secrets/kubernetes.io/serviceaccount/token")
if err != nil {
return env.Get(ConsoleOperatorSAToken, "")
}
return string(dat)
}
func getConsoleImage() string {
once.Do(func() {
initVars()
})
return consoleImage
}

2
restapi/configure_console.go
View File

@@ -121,6 +121,8 @@ func configureAPI(api *operations.ConsoleAPI) http.Handler {
registerServiceAccountsHandlers(api)
// Register admin remote buckets
registerAdminBucketRemoteHandlers(api)
// Register admin log search
registerLogSearchHandlers(api)
// Register admin subscription handlers
registerSubscriptionHandlers(api)

9
restapi/consts.go
View File

@@ -48,6 +48,11 @@ const (
ConsoleSecureReferrerPolicy = "CONSOLE_SECURE_REFERRER_POLICY"
ConsoleSecureFeaturePolicy = "CONSOLE_SECURE_FEATURE_POLICY"
ConsoleSecureExpectCTHeader = "CONSOLE_SECURE_EXPECT_CT_HEADER"
ConsoleOperatorSAToken = "CONSOLE_OPERATOR_SA_TOKEN"
ConsoleOperatorConsoleImage = "CONSOLE_OPERATOR_CONSOLE_IMAGE"
LogSearchURL = "CONSOLE_LOG_QUERY_URL"
PrometheusURL = "CONSOLE_PROMETHEUS_URL"
LogSearchQueryAuthToken = "LOGSEARCH_QUERY_AUTH_TOKEN"
// Constants for prometheus annotations
prometheusPath = "prometheus.io/path"
@@ -57,8 +62,8 @@ const (
// Image versions
const (
KESImageVersion = "minio/kes:v0.12.1"
ConsoleImageVersion = "minio/console:v0.4.6"
KESImageVersion = "minio/kes:v0.13.1"
ConsoleImageDefaultVersion = "minio/console:v0.4.6"
)
// K8s

396
restapi/embedded_spec.go
View File

@@ -115,6 +115,24 @@ func init() {
],
"summary": "Returns information about the deployment",
"operationId": "AdminInfo",
"parameters": [
{
"type": "integer",
"name": "start",
"in": "query"
},
{
"type": "integer",
"name": "end",
"in": "query"
},
{
"type": "integer",
"default": 15,
"name": "step",
"in": "query"
}
],
"responses": {
"200": {
"description": "A successful response.",
@@ -1713,6 +1731,70 @@ func init() {
}
}
},
"/logs/search": {
"get": {
"tags": [
"UserAPI"
],
"summary": "Search the logs",
"operationId": "LogSearch",
"parameters": [
{
"type": "array",
"items": {
"type": "string"
},
"collectionFormat": "multi",
"description": "Filter Parameters",
"name": "fp",
"in": "query"
},
{
"type": "number",
"format": "int32",
"default": 10,
"name": "pageSize",
"in": "query"
},
{
"type": "number",
"format": "int32",
"default": 0,
"name": "pageNo",
"in": "query"
},
{
"enum": [
"timeDesc",
"timeAsc"
],
"type": "string",
"default": "timeDesc",
"name": "order",
"in": "query"
},
{
"type": "string",
"name": "timeStart",
"in": "query"
}
],
"responses": {
"200": {
"description": "A successful response.",
"schema": {
"$ref": "#/definitions/logSearchResponse"
}
},
"default": {
"description": "Generic error response.",
"schema": {
"$ref": "#/definitions/error"
}
}
}
}
},
"/namespaces/{namespace}/resourcequotas/{resource-quota-name}": {
"get": {
"tags": [
@@ -3119,6 +3201,12 @@ func init() {
},
"usage": {
"type": "integer"
},
"widgets": {
"type": "array",
"items": {
"$ref": "#/definitions/widget"
}
}
}
},
@@ -3517,6 +3605,12 @@ func init() {
"erasureCodingParity": {
"type": "integer"
},
"expose_console": {
"type": "boolean"
},
"expose_minio": {
"type": "boolean"
},
"idp": {
"type": "object",
"$ref": "#/definitions/idpConfiguration"
@@ -3985,6 +4079,15 @@ func init() {
}
}
},
"logSearchResponse": {
"type": "object",
"properties": {
"results": {
"type": "object",
"title": "list of log search responses"
}
}
},
"loginDetails": {
"type": "object",
"properties": {
@@ -4838,6 +4941,23 @@ func init() {
}
}
},
"resultTarget": {
"type": "object",
"properties": {
"legendFormat": {
"type": "string"
},
"result": {
"type": "array",
"items": {
"$ref": "#/definitions/widgetResult"
}
},
"resultType": {
"type": "string"
}
}
},
"serviceAccountCreds": {
"type": "object",
"properties": {
@@ -5071,6 +5191,17 @@ func init() {
"enable_prometheus": {
"type": "boolean"
},
"endpoints": {
"type": "object",
"properties": {
"console": {
"type": "string"
},
"minio": {
"type": "string"
}
}
},
"image": {
"type": "string"
},
@@ -5312,6 +5443,54 @@ func init() {
}
}
}
},
"widget": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"reduceOptions": {
"type": "object",
"properties": {
"calcs": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
}
},
"targets": {
"type": "array",
"items": {
"$ref": "#/definitions/resultTarget"
}
},
"title": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"widgetResult": {
"type": "object",
"properties": {
"metric": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"values": {
"type": "array",
"items": {}
}
}
}
},
"securityDefinitions": {
@@ -5409,6 +5588,24 @@ func init() {
],
"summary": "Returns information about the deployment",
"operationId": "AdminInfo",
"parameters": [
{
"type": "integer",
"name": "start",
"in": "query"
},
{
"type": "integer",
"name": "end",
"in": "query"
},
{
"type": "integer",
"default": 15,
"name": "step",
"in": "query"
}
],
"responses": {
"200": {
"description": "A successful response.",
@@ -7007,6 +7204,70 @@ func init() {
}
}
},
"/logs/search": {
"get": {
"tags": [
"UserAPI"
],
"summary": "Search the logs",
"operationId": "LogSearch",
"parameters": [
{
"type": "array",
"items": {
"type": "string"
},
"collectionFormat": "multi",
"description": "Filter Parameters",
"name": "fp",
"in": "query"
},
{
"type": "number",
"format": "int32",
"default": 10,
"name": "pageSize",
"in": "query"
},
{
"type": "number",
"format": "int32",
"default": 0,
"name": "pageNo",
"in": "query"
},
{
"enum": [
"timeDesc",
"timeAsc"
],
"type": "string",
"default": "timeDesc",
"name": "order",
"in": "query"
},
{
"type": "string",
"name": "timeStart",
"in": "query"
}
],
"responses": {
"200": {
"description": "A successful response.",
"schema": {
"$ref": "#/definitions/logSearchResponse"
}
},
"default": {
"description": "Generic error response.",
"schema": {
"$ref": "#/definitions/error"
}
}
}
}
},
"/namespaces/{namespace}/resourcequotas/{resource-quota-name}": {
"get": {
"tags": [
@@ -8799,6 +9060,17 @@ func init() {
}
}
},
"TenantEndpoints": {
"type": "object",
"properties": {
"console": {
"type": "string"
},
"minio": {
"type": "string"
}
}
},
"VaultConfigurationApprole": {
"type": "object",
"required": [
@@ -8844,6 +9116,33 @@ func init() {
}
}
},
"WidgetOptions": {
"type": "object",
"properties": {
"reduceOptions": {
"type": "object",
"properties": {
"calcs": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
}
},
"WidgetOptionsReduceOptions": {
"type": "object",
"properties": {
"calcs": {
"type": "array",
"items": {
"type": "string"
}
}
}
},
"accountChangePasswordRequest": {
"type": "object",
"required": [
@@ -8936,6 +9235,12 @@ func init() {
},
"usage": {
"type": "integer"
},
"widgets": {
"type": "array",
"items": {
"$ref": "#/definitions/widget"
}
}
}
},
@@ -9334,6 +9639,12 @@ func init() {
"erasureCodingParity": {
"type": "integer"
},
"expose_console": {
"type": "boolean"
},
"expose_minio": {
"type": "boolean"
},
"idp": {
"type": "object",
"$ref": "#/definitions/idpConfiguration"
@@ -9802,6 +10113,15 @@ func init() {
}
}
},
"logSearchResponse": {
"type": "object",
"properties": {
"results": {
"type": "object",
"title": "list of log search responses"
}
}
},
"loginDetails": {
"type": "object",
"properties": {
@@ -10520,6 +10840,23 @@ func init() {
}
}
},
"resultTarget": {
"type": "object",
"properties": {
"legendFormat": {
"type": "string"
},
"result": {
"type": "array",
"items": {
"$ref": "#/definitions/widgetResult"
}
},
"resultType": {
"type": "string"
}
}
},
"serviceAccountCreds": {
"type": "object",
"properties": {
@@ -10753,6 +11090,17 @@ func init() {
"enable_prometheus": {
"type": "boolean"
},
"endpoints": {
"type": "object",
"properties": {
"console": {
"type": "string"
},
"minio": {
"type": "string"
}
}
},
"image": {
"type": "string"
},
@@ -10994,6 +11342,54 @@ func init() {
}
}
}
},
"widget": {
"type": "object",
"properties": {
"options": {
"type": "object",
"properties": {
"reduceOptions": {
"type": "object",
"properties": {
"calcs": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
}
},
"targets": {
"type": "array",
"items": {
"$ref": "#/definitions/resultTarget"
}
},
"title": {
"type": "string"
},
"type": {
"type": "string"
}
}
},
"widgetResult": {
"type": "object",
"properties": {
"metric": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"values": {
"type": "array",
"items": {}
}
}
}
},
"securityDefinitions": {

113
restapi/operations/admin_api/admin_info_parameters.go
View File

@@ -26,14 +26,25 @@ import (
"net/http"
"github.com/go-openapi/errors"
"github.com/go-openapi/runtime"
"github.com/go-openapi/runtime/middleware"
"github.com/go-openapi/strfmt"
"github.com/go-openapi/swag"
)
// NewAdminInfoParams creates a new AdminInfoParams object
// no default values defined in spec.
// with the default values initialized.
func NewAdminInfoParams() AdminInfoParams {
return AdminInfoParams{}
var (
// initialize parameters with default values
stepDefault = int64(15)
)
return AdminInfoParams{
Step: &stepDefault,
}
}
// AdminInfoParams contains all the bound params for the admin info operation
@@ -44,6 +55,20 @@ type AdminInfoParams struct {
// HTTP Request Object
HTTPRequest *http.Request `json:"-"`
/*
In: query
*/
End *int64
/*
In: query
*/
Start *int64
/*
In: query
Default: 15
*/
Step *int64
}
// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface
@@ -55,8 +80,92 @@ func (o *AdminInfoParams) BindRequest(r *http.Request, route *middleware.Matched
o.HTTPRequest = r
qs := runtime.Values(r.URL.Query())
qEnd, qhkEnd, _ := qs.GetOK("end")
if err := o.bindEnd(qEnd, qhkEnd, route.Formats); err != nil {
res = append(res, err)
}
qStart, qhkStart, _ := qs.GetOK("start")
if err := o.bindStart(qStart, qhkStart, route.Formats); err != nil {
res = append(res, err)
}
qStep, qhkStep, _ := qs.GetOK("step")
if err := o.bindStep(qStep, qhkStep, route.Formats); err != nil {
res = append(res, err)
}
if len(res) > 0 {
return errors.CompositeValidationError(res...)
}
return nil
}
// bindEnd binds and validates parameter End from query.
func (o *AdminInfoParams) bindEnd(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
return nil
}
value, err := swag.ConvertInt64(raw)
if err != nil {
return errors.InvalidType("end", "query", "int64", raw)
}
o.End = &value
return nil
}
// bindStart binds and validates parameter Start from query.
func (o *AdminInfoParams) bindStart(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
return nil
}
value, err := swag.ConvertInt64(raw)
if err != nil {
return errors.InvalidType("start", "query", "int64", raw)
}
o.Start = &value
return nil
}
// bindStep binds and validates parameter Step from query.
func (o *AdminInfoParams) bindStep(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
// Default values have been previously initialized by NewAdminInfoParams()
return nil
}
value, err := swag.ConvertInt64(raw)
if err != nil {
return errors.InvalidType("step", "query", "int64", raw)
}
o.Step = &value
return nil
}

36
restapi/operations/admin_api/admin_info_urlbuilder.go
View File

@@ -26,11 +26,19 @@ import (
"errors"
"net/url"
golangswaggerpaths "path"
"github.com/go-openapi/swag"
)
// AdminInfoURL generates an URL for the admin info operation
type AdminInfoURL struct {
End *int64
Start *int64
Step *int64
_basePath string
// avoid unkeyed usage
_ struct{}
}
// WithBasePath sets the base path for this url builder, only required when it's different from the
@@ -60,6 +68,34 @@ func (o *AdminInfoURL) Build() (*url.URL, error) {
}
_result.Path = golangswaggerpaths.Join(_basePath, _path)
qs := make(url.Values)
var endQ string
if o.End != nil {
endQ = swag.FormatInt64(*o.End)
}
if endQ != "" {
qs.Set("end", endQ)
}
var startQ string
if o.Start != nil {
startQ = swag.FormatInt64(*o.Start)
}
if startQ != "" {
qs.Set("start", startQ)
}
var stepQ string
if o.Step != nil {
stepQ = swag.FormatInt64(*o.Step)
}
if stepQ != "" {
qs.Set("step", stepQ)
}
_result.RawQuery = qs.Encode()
return &_result, nil
}

12
restapi/operations/console_api.go
View File

@@ -208,6 +208,9 @@ func NewConsoleAPI(spec *loads.Document) *ConsoleAPI {
AdminAPIListUsersHandler: admin_api.ListUsersHandlerFunc(func(params admin_api.ListUsersParams, principal *models.Principal) middleware.Responder {
return middleware.NotImplemented("operation admin_api.ListUsers has not yet been implemented")
}),
UserAPILogSearchHandler: user_api.LogSearchHandlerFunc(func(params user_api.LogSearchParams, principal *models.Principal) middleware.Responder {
return middleware.NotImplemented("operation user_api.LogSearch has not yet been implemented")
}),
UserAPILoginHandler: user_api.LoginHandlerFunc(func(params user_api.LoginParams) middleware.Responder {
return middleware.NotImplemented("operation user_api.Login has not yet been implemented")
}),
@@ -473,6 +476,8 @@ type ConsoleAPI struct {
UserAPIListUserServiceAccountsHandler user_api.ListUserServiceAccountsHandler
// AdminAPIListUsersHandler sets the operation handler for the list users operation
AdminAPIListUsersHandler admin_api.ListUsersHandler
// UserAPILogSearchHandler sets the operation handler for the log search operation
UserAPILogSearchHandler user_api.LogSearchHandler
// UserAPILoginHandler sets the operation handler for the login operation
UserAPILoginHandler user_api.LoginHandler
// UserAPILoginDetailHandler sets the operation handler for the login detail operation
@@ -771,6 +776,9 @@ func (o *ConsoleAPI) Validate() error {
if o.AdminAPIListUsersHandler == nil {
unregistered = append(unregistered, "admin_api.ListUsersHandler")
}
if o.UserAPILogSearchHandler == nil {
unregistered = append(unregistered, "user_api.LogSearchHandler")
}
if o.UserAPILoginHandler == nil {
unregistered = append(unregistered, "user_api.LoginHandler")
}
@@ -1182,6 +1190,10 @@ func (o *ConsoleAPI) initHandlerCache() {
o.handlers["GET"] = make(map[string]http.Handler)
}
o.handlers["GET"]["/users"] = admin_api.NewListUsers(o.context, o.AdminAPIListUsersHandler)
if o.handlers["GET"] == nil {
o.handlers["GET"] = make(map[string]http.Handler)
}
o.handlers["GET"]["/logs/search"] = user_api.NewLogSearch(o.context, o.UserAPILogSearchHandler)
if o.handlers["POST"] == nil {
o.handlers["POST"] = make(map[string]http.Handler)
}

90
restapi/operations/user_api/log_search.go Normal file
View File

@@ -0,0 +1,90 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package user_api
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the generate command
import (
"net/http"
"github.com/go-openapi/runtime/middleware"
"github.com/minio/console/models"
)
// LogSearchHandlerFunc turns a function with the right signature into a log search handler
type LogSearchHandlerFunc func(LogSearchParams, *models.Principal) middleware.Responder
// Handle executing the request and returning a response
func (fn LogSearchHandlerFunc) Handle(params LogSearchParams, principal *models.Principal) middleware.Responder {
return fn(params, principal)
}
// LogSearchHandler interface for that can handle valid log search params
type LogSearchHandler interface {
Handle(LogSearchParams, *models.Principal) middleware.Responder
}
// NewLogSearch creates a new http.Handler for the log search operation
func NewLogSearch(ctx *middleware.Context, handler LogSearchHandler) *LogSearch {
return &LogSearch{Context: ctx, Handler: handler}
}
/*LogSearch swagger:route GET /logs/search UserAPI logSearch
Search the logs
*/
type LogSearch struct {
Context *middleware.Context
Handler LogSearchHandler
}
func (o *LogSearch) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
route, rCtx, _ := o.Context.RouteInfo(r)
if rCtx != nil {
r = rCtx
}
var Params = NewLogSearchParams()
uprinc, aCtx, err := o.Context.Authorize(r, route)
if err != nil {
o.Context.Respond(rw, r, route.Produces, route, err)
return
}
if aCtx != nil {
r = aCtx
}
var principal *models.Principal
if uprinc != nil {
principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise
}
if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params
o.Context.Respond(rw, r, route.Produces, route, err)
return
}
res := o.Handler.Handle(Params, principal) // actually handle the request
o.Context.Respond(rw, r, route.Produces, route, res)
}

253
restapi/operations/user_api/log_search_parameters.go Normal file
View File

@@ -0,0 +1,253 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package user_api
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the swagger generate command
import (
"net/http"
"github.com/go-openapi/errors"
"github.com/go-openapi/runtime"
"github.com/go-openapi/runtime/middleware"
"github.com/go-openapi/strfmt"
"github.com/go-openapi/swag"
"github.com/go-openapi/validate"
)
// NewLogSearchParams creates a new LogSearchParams object
// with the default values initialized.
func NewLogSearchParams() LogSearchParams {
var (
// initialize parameters with default values
orderDefault = string("timeDesc")
pageNoDefault = int32(0)
pageSizeDefault = int32(10)
)
return LogSearchParams{
Order: &orderDefault,
PageNo: &pageNoDefault,
PageSize: &pageSizeDefault,
}
}
// LogSearchParams contains all the bound params for the log search operation
// typically these are obtained from a http.Request
//
// swagger:parameters LogSearch
type LogSearchParams struct {
// HTTP Request Object
HTTPRequest *http.Request `json:"-"`
/*Filter Parameters
In: query
Collection Format: multi
*/
Fp []string
/*
In: query
Default: "timeDesc"
*/
Order *string
/*
In: query
Default: 0
*/
PageNo *int32
/*
In: query
Default: 10
*/
PageSize *int32
/*
In: query
*/
TimeStart *string
}
// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface
// for simple values it will use straight method calls.
//
// To ensure default values, the struct must have been initialized with NewLogSearchParams() beforehand.
func (o *LogSearchParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error {
var res []error
o.HTTPRequest = r
qs := runtime.Values(r.URL.Query())
qFp, qhkFp, _ := qs.GetOK("fp")
if err := o.bindFp(qFp, qhkFp, route.Formats); err != nil {
res = append(res, err)
}
qOrder, qhkOrder, _ := qs.GetOK("order")
if err := o.bindOrder(qOrder, qhkOrder, route.Formats); err != nil {
res = append(res, err)
}
qPageNo, qhkPageNo, _ := qs.GetOK("pageNo")
if err := o.bindPageNo(qPageNo, qhkPageNo, route.Formats); err != nil {
res = append(res, err)
}
qPageSize, qhkPageSize, _ := qs.GetOK("pageSize")
if err := o.bindPageSize(qPageSize, qhkPageSize, route.Formats); err != nil {
res = append(res, err)
}
qTimeStart, qhkTimeStart, _ := qs.GetOK("timeStart")
if err := o.bindTimeStart(qTimeStart, qhkTimeStart, route.Formats); err != nil {
res = append(res, err)
}
if len(res) > 0 {
return errors.CompositeValidationError(res...)
}
return nil
}
// bindFp binds and validates array parameter Fp from query.
//
// Arrays are parsed according to CollectionFormat: "multi" (defaults to "csv" when empty).
func (o *LogSearchParams) bindFp(rawData []string, hasKey bool, formats strfmt.Registry) error {
// CollectionFormat: multi
fpIC := rawData
if len(fpIC) == 0 {
return nil
}
var fpIR []string
for _, fpIV := range fpIC {
fpI := fpIV
fpIR = append(fpIR, fpI)
}
o.Fp = fpIR
return nil
}
// bindOrder binds and validates parameter Order from query.
func (o *LogSearchParams) bindOrder(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
// Default values have been previously initialized by NewLogSearchParams()
return nil
}
o.Order = &raw
if err := o.validateOrder(formats); err != nil {
return err
}
return nil
}
// validateOrder carries on validations for parameter Order
func (o *LogSearchParams) validateOrder(formats strfmt.Registry) error {
if err := validate.EnumCase("order", "query", *o.Order, []interface{}{"timeDesc", "timeAsc"}, true); err != nil {
return err
}
return nil
}
// bindPageNo binds and validates parameter PageNo from query.
func (o *LogSearchParams) bindPageNo(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
// Default values have been previously initialized by NewLogSearchParams()
return nil
}
value, err := swag.ConvertInt32(raw)
if err != nil {
return errors.InvalidType("pageNo", "query", "int32", raw)
}
o.PageNo = &value
return nil
}
// bindPageSize binds and validates parameter PageSize from query.
func (o *LogSearchParams) bindPageSize(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
// Default values have been previously initialized by NewLogSearchParams()
return nil
}
value, err := swag.ConvertInt32(raw)
if err != nil {
return errors.InvalidType("pageSize", "query", "int32", raw)
}
o.PageSize = &value
return nil
}
// bindTimeStart binds and validates parameter TimeStart from query.
func (o *LogSearchParams) bindTimeStart(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
return nil
}
o.TimeStart = &raw
return nil
}

133
restapi/operations/user_api/log_search_responses.go Normal file
View File

@@ -0,0 +1,133 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package user_api
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the swagger generate command
import (
"net/http"
"github.com/go-openapi/runtime"
"github.com/minio/console/models"
)
// LogSearchOKCode is the HTTP code returned for type LogSearchOK
const LogSearchOKCode int = 200
/*LogSearchOK A successful response.
swagger:response logSearchOK
*/
type LogSearchOK struct {
/*
In: Body
*/
Payload *models.LogSearchResponse `json:"body,omitempty"`
}
// NewLogSearchOK creates LogSearchOK with default headers values
func NewLogSearchOK() *LogSearchOK {
return &LogSearchOK{}
}
// WithPayload adds the payload to the log search o k response
func (o *LogSearchOK) WithPayload(payload *models.LogSearchResponse) *LogSearchOK {
o.Payload = payload
return o
}
// SetPayload sets the payload to the log search o k response
func (o *LogSearchOK) SetPayload(payload *models.LogSearchResponse) {
o.Payload = payload
}
// WriteResponse to the client
func (o *LogSearchOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) {
rw.WriteHeader(200)
if o.Payload != nil {
payload := o.Payload
if err := producer.Produce(rw, payload); err != nil {
panic(err) // let the recovery middleware deal with this
}
}
}
/*LogSearchDefault Generic error response.
swagger:response logSearchDefault
*/
type LogSearchDefault struct {
_statusCode int
/*
In: Body
*/
Payload *models.Error `json:"body,omitempty"`
}
// NewLogSearchDefault creates LogSearchDefault with default headers values
func NewLogSearchDefault(code int) *LogSearchDefault {
if code <= 0 {
code = 500
}
return &LogSearchDefault{
_statusCode: code,
}
}
// WithStatusCode adds the status to the log search default response
func (o *LogSearchDefault) WithStatusCode(code int) *LogSearchDefault {
o._statusCode = code
return o
}
// SetStatusCode sets the status to the log search default response
func (o *LogSearchDefault) SetStatusCode(code int) {
o._statusCode = code
}
// WithPayload adds the payload to the log search default response
func (o *LogSearchDefault) WithPayload(payload *models.Error) *LogSearchDefault {
o.Payload = payload
return o
}
// SetPayload sets the payload to the log search default response
func (o *LogSearchDefault) SetPayload(payload *models.Error) {
o.Payload = payload
}
// WriteResponse to the client
func (o *LogSearchDefault) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) {
rw.WriteHeader(o._statusCode)
if o.Payload != nil {
payload := o.Payload
if err := producer.Produce(rw, payload); err != nil {
panic(err) // let the recovery middleware deal with this
}
}
}

164
restapi/operations/user_api/log_search_urlbuilder.go Normal file
View File

@@ -0,0 +1,164 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package user_api
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the generate command
import (
"errors"
"net/url"
golangswaggerpaths "path"
"github.com/go-openapi/swag"
)
// LogSearchURL generates an URL for the log search operation
type LogSearchURL struct {
Fp []string
Order *string
PageNo *int32
PageSize *int32
TimeStart *string
_basePath string
// avoid unkeyed usage
_ struct{}
}
// WithBasePath sets the base path for this url builder, only required when it's different from the
// base path specified in the swagger spec.
// When the value of the base path is an empty string
func (o *LogSearchURL) WithBasePath(bp string) *LogSearchURL {
o.SetBasePath(bp)
return o
}
// SetBasePath sets the base path for this url builder, only required when it's different from the
// base path specified in the swagger spec.
// When the value of the base path is an empty string
func (o *LogSearchURL) SetBasePath(bp string) {
o._basePath = bp
}
// Build a url path and query string
func (o *LogSearchURL) Build() (*url.URL, error) {
var _result url.URL
var _path = "/logs/search"
_basePath := o._basePath
if _basePath == "" {
_basePath = "/api/v1"
}
_result.Path = golangswaggerpaths.Join(_basePath, _path)
qs := make(url.Values)
var fpIR []string
for _, fpI := range o.Fp {
fpIS := fpI
if fpIS != "" {
fpIR = append(fpIR, fpIS)
}
}
fp := swag.JoinByFormat(fpIR, "multi")
for _, qsv := range fp {
qs.Add("fp", qsv)
}
var orderQ string
if o.Order != nil {
orderQ = *o.Order
}
if orderQ != "" {
qs.Set("order", orderQ)
}
var pageNoQ string
if o.PageNo != nil {
pageNoQ = swag.FormatInt32(*o.PageNo)
}
if pageNoQ != "" {
qs.Set("pageNo", pageNoQ)
}
var pageSizeQ string
if o.PageSize != nil {
pageSizeQ = swag.FormatInt32(*o.PageSize)
}
if pageSizeQ != "" {
qs.Set("pageSize", pageSizeQ)
}
var timeStartQ string
if o.TimeStart != nil {
timeStartQ = *o.TimeStart
}
if timeStartQ != "" {
qs.Set("timeStart", timeStartQ)
}
_result.RawQuery = qs.Encode()
return &_result, nil
}
// Must is a helper function to panic when the url builder returns an error
func (o *LogSearchURL) Must(u *url.URL, err error) *url.URL {
if err != nil {
panic(err)
}
if u == nil {
panic("url can't be nil")
}
return u
}
// String returns the string representation of the path with query string
func (o *LogSearchURL) String() string {
return o.Must(o.Build()).String()
}
// BuildFull builds a full url with scheme, host, path and query string
func (o *LogSearchURL) BuildFull(scheme, host string) (*url.URL, error) {
if scheme == "" {
return nil, errors.New("scheme is required for a full url on LogSearchURL")
}
if host == "" {
return nil, errors.New("host is required for a full url on LogSearchURL")
}
base, err := o.Build()
if err != nil {
return nil, err
}
base.Scheme = scheme
base.Host = host
return base, nil
}
// StringFull returns the string representation of a complete url
func (o *LogSearchURL) StringFull(scheme, host string) string {
return o.Must(o.BuildFull(scheme, host)).String()
}

2
restapi/user_buckets.go
View File

@@ -272,7 +272,7 @@ func getAccountInfo(ctx context.Context, client MinioAdmin) ([]*models.Bucket, e
}
var bucketInfos []*models.Bucket
for _, bucket := range info.Buckets {
bucketElem := &models.Bucket{Name: swag.String(bucket.Name), CreationDate: bucket.Created.String(), Size: int64(bucket.Size)}
bucketElem := &models.Bucket{Name: swag.String(bucket.Name), CreationDate: bucket.Created.Format(time.RFC3339), Size: int64(bucket.Size)}
bucketInfos = append(bucketInfos, bucketElem)
}
return bucketInfos, nil

2
restapi/user_buckets_test.go
View File

@@ -128,7 +128,7 @@ func TestListBucket(t *testing.T) {
assert.Equal(len(mockBucketList.Buckets), len(bucketList), fmt.Sprintf("Failed on %s: length of bucket's lists is not the same", function))
for i, b := range bucketList {
assert.Equal(mockBucketList.Buckets[i].Name, *b.Name)
assert.Equal(mockBucketList.Buckets[i].Created.String(), b.CreationDate)
assert.Equal(mockBucketList.Buckets[i].Created.Format(time.RFC3339), b.CreationDate)
assert.Equal(mockBucketList.Buckets[i].Name, *b.Name)
assert.Equal(int64(mockBucketList.Buckets[i].Size), b.Size)
}

100
restapi/user_log_search.go Normal file
View File

@@ -0,0 +1,100 @@
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package restapi
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"github.com/go-openapi/swag"
"github.com/go-openapi/runtime/middleware"
"github.com/minio/console/models"
"github.com/minio/console/restapi/operations"
"github.com/minio/console/restapi/operations/user_api"
logsearchServer "github.com/minio/operator/logsearchapi/server"
)
func registerLogSearchHandlers(api *operations.ConsoleAPI) {
// log search
api.UserAPILogSearchHandler = user_api.LogSearchHandlerFunc(func(params user_api.LogSearchParams, session *models.Principal) middleware.Responder {
searchResp, err := getLogSearchResponse(params)
if err != nil {
return user_api.NewLogSearchDefault(int(err.Code)).WithPayload(err)
}
return user_api.NewLogSearchOK().WithPayload(searchResp)
})
}
func init() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
}
// getLogSearchResponse performs a query to Log Search if Enabled
func getLogSearchResponse(params user_api.LogSearchParams) (*models.LogSearchResponse, *models.Error) {
token := getLogSearchAPIToken()
endpoint := fmt.Sprintf("%s/api/query?token=%s&q=reqinfo", getLogSearchURL(), token)
for _, fp := range params.Fp {
endpoint = fmt.Sprintf("%s&fp=%s", endpoint, fp)
}
endpoint = fmt.Sprintf("%s&%s=ok", endpoint, *params.Order)
// timeStart
if params.TimeStart != nil && *params.TimeStart != "" {
endpoint = fmt.Sprintf("%s&timeStart=%s", endpoint, *params.TimeStart)
}
// page size and page number
endpoint = fmt.Sprintf("%s&pageSize=%d", endpoint, *params.PageSize)
endpoint = fmt.Sprintf("%s&pageNo=%d", endpoint, *params.PageNo)
return logSearch(endpoint)
}
func logSearch(endpoint string) (*models.LogSearchResponse, *models.Error) {
resp, err := http.Get(endpoint)
if err != nil {
return nil, prepareError(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
log.Println("Error Status Code", resp.StatusCode)
_, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, prepareError(err)
}
return nil, &models.Error{
Code: 500,
Message: swag.String("Error retrieving logs"),
}
}
var results []logsearchServer.ReqInfoRow
if err := json.NewDecoder(resp.Body).Decode(&results); err != nil {
return nil, prepareError(err)
}
response := models.LogSearchResponse{
Results: results,
}
return &response, nil
}

129
restapi/user_log_search_test.go Normal file
View File

@@ -0,0 +1,129 @@
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package restapi
import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"reflect"
"testing"
"time"
"github.com/go-openapi/swag"
"github.com/minio/console/models"
logsearchServer "github.com/minio/operator/logsearchapi/server"
asrt "github.com/stretchr/testify/assert"
)
func TestLogSearch(t *testing.T) {
responseItem := []logsearchServer.ReqInfoRow{
{
Time: time.Time{},
APIName: "GetConfigKV",
Bucket: "",
Object: "",
TimeToResponseNs: 45254653,
RemoteHost: "10.116.1.94",
RequestID: "16595A4E30CCFE79",
UserAgent: "MinIO (linux; amd64) madmin-go/0.0.1",
ResponseStatus: "OK",
ResponseStatusCode: 200,
RequestContentLength: nil,
ResponseContentLength: nil,
}, {
Time: time.Time{},
APIName: "AssumeRole",
Bucket: "",
Object: "",
TimeToResponseNs: 307423794,
RemoteHost: "127.0.0.1",
RequestID: "16595A4DA906FBA9",
UserAgent: "Go-http-client/1.1",
ResponseStatus: "OK",
ResponseStatusCode: 200,
RequestContentLength: nil,
ResponseContentLength: nil,
},
}
assert := asrt.New(t)
type args struct {
apiResponse string
apiResponseCode int
}
response, _ := json.Marshal(responseItem)
successfulResponse := &models.LogSearchResponse{
Results: responseItem,
}
tests := []struct {
name string
args args
expectedResponse *models.LogSearchResponse
expectedError *models.Error
}{
{
name: "200 Success response",
args: args{
apiResponse: fmt.Sprintf("%s\n", response),
apiResponseCode: 200,
},
expectedResponse: successfulResponse,
expectedError: nil,
},
{
name: "500 unsuccessful response",
args: args{
apiResponse: "Some random error",
apiResponseCode: 500,
},
expectedResponse: nil,
expectedError: &models.Error{
Code: 500,
Message: swag.String("Error retrieving logs"),
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
testRequest := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(tt.args.apiResponseCode)
fmt.Fprintln(w, tt.args.apiResponse)
}))
defer testRequest.Close()
resp, err := logSearch(testRequest.URL)
if tt.expectedError != nil {
fmt.Println(t.Name())
assert.Equal(tt.expectedError.Code, err.Code, fmt.Sprintf("logSearch() error code: `%v`, wantErr: `%v`", err.Code, tt.expectedError))
assert.Equal(tt.expectedError.Message, err.Message, fmt.Sprintf("logSearch() error message: `%v`, wantErr: `%v`", err.Message, tt.expectedError))
} else {
assert.Nil(err, fmt.Sprintf("logSearch() error: %v, wantErr: %v", err, tt.expectedError))
if !reflect.DeepEqual(resp, tt.expectedResponse) {
t.Errorf("logSearch() resp: %v, expectedResponse: %v", resp, tt.expectedResponse)
return
}
}
})
}
}

25
restapi/user_login.go
View File

@@ -176,9 +176,8 @@ func getLoginDetailsResponse() (*models.LoginDetails, *models.Error) {
defer cancel()
loginStrategy := models.LoginDetailsLoginStrategyForm
redirectURL := ""
if acl.GetOperatorMode() {
loginStrategy = models.LoginDetailsLoginStrategyServiceAccount
} else if oauth2.IsIdpEnabled() {
if oauth2.IsIdpEnabled() {
loginStrategy = models.LoginDetailsLoginStrategyRedirect
// initialize new oauth2 client
oauth2Client, err := oauth2.NewOauth2ProviderClient(ctx, nil)
@@ -188,7 +187,10 @@ func getLoginDetailsResponse() (*models.LoginDetails, *models.Error) {
// Validate user against IDP
identityProvider := &auth.IdentityProvider{Client: oauth2Client}
redirectURL = identityProvider.GenerateLoginURL()
} else if acl.GetOperatorMode() {
loginStrategy = models.LoginDetailsLoginStrategyServiceAccount
}
loginDetails := &models.LoginDetails{
LoginStrategy: loginStrategy,
Redirect: redirectURL,
@@ -209,7 +211,22 @@ func verifyUserAgainstIDP(ctx context.Context, provider auth.IdentityProviderI,
func getLoginOauth2AuthResponse(lr *models.LoginOauth2AuthRequest) (*models.LoginResponse, *models.Error) {
ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second)
defer cancel()
if oauth2.IsIdpEnabled() {
if acl.GetOperatorMode() {
creds, err := newConsoleCredentials("", getK8sSAToken(), "")
if err != nil {
return nil, prepareError(err)
}
credentials := consoleCredentials{consoleCredentials: creds, actions: []string{}}
token, err := login(credentials)
if err != nil {
return nil, prepareError(errInvalidCredentials, nil, err)
}
// serialize output
loginResponse := &models.LoginResponse{
SessionID: *token,
}
return loginResponse, nil
} else if oauth2.IsIdpEnabled() {
// initialize new oauth2 client
oauth2Client, err := oauth2.NewOauth2ProviderClient(ctx, nil)
if err != nil {

4
restapi/user_objects.go
View File

@@ -173,7 +173,7 @@ func listBucketObjects(ctx context.Context, client MinioClient, bucketName strin
obj := &models.BucketObject{
Name: lsObj.Key,
Size: lsObj.Size,
LastModified: lsObj.LastModified.String(),
LastModified: lsObj.LastModified.Format(time.RFC3339),
ContentType: lsObj.ContentType,
VersionID: lsObj.VersionID,
IsLatest: lsObj.IsLatest,
@@ -205,7 +205,7 @@ func listBucketObjects(ctx context.Context, client MinioClient, bucketName strin
if retention != nil && retUntilDate != nil {
date := *retUntilDate
obj.RetentionMode = string(*retention)
obj.RetentionUntilDate = date.String()
obj.RetentionUntilDate = date.Format(time.RFC3339)
}
}
tags, err := client.getObjectTagging(ctx, bucketName, lsObj.Key, minio.GetObjectTaggingOptions{VersionID: lsObj.VersionID})

24
restapi/user_objects_test.go
View File

@@ -170,23 +170,23 @@ func Test_listObjects(t *testing.T) {
expectedResp: []*models.BucketObject{
&models.BucketObject{
Name: "obj1",
LastModified: t1.String(),
LastModified: t1.Format(time.RFC3339),
Size: int64(1024),
ContentType: "content",
LegalHoldStatus: string(minio.LegalHoldEnabled),
RetentionMode: string(minio.Governance),
RetentionUntilDate: tretention.String(),
RetentionUntilDate: tretention.Format(time.RFC3339),
Tags: map[string]string{
"tag1": "value1",
},
}, &models.BucketObject{
Name: "obj2",
LastModified: t1.String(),
LastModified: t1.Format(time.RFC3339),
Size: int64(512),
ContentType: "content",
LegalHoldStatus: string(minio.LegalHoldEnabled),
RetentionMode: string(minio.Governance),
RetentionUntilDate: tretention.String(),
RetentionUntilDate: tretention.Format(time.RFC3339),
Tags: map[string]string{
"tag1": "value1",
},
@@ -332,18 +332,18 @@ func Test_listObjects(t *testing.T) {
expectedResp: []*models.BucketObject{
&models.BucketObject{
Name: "obj1",
LastModified: t1.String(),
LastModified: t1.Format(time.RFC3339),
Size: int64(1024),
ContentType: "content",
IsDeleteMarker: true,
}, &models.BucketObject{
Name: "obj2",
LastModified: t1.String(),
LastModified: t1.Format(time.RFC3339),
Size: int64(512),
ContentType: "content",
LegalHoldStatus: string(minio.LegalHoldEnabled),
RetentionMode: string(minio.Governance),
RetentionUntilDate: tretention.String(),
RetentionUntilDate: tretention.Format(time.RFC3339),
Tags: map[string]string{
"tag1": "value1",
},
@@ -391,7 +391,7 @@ func Test_listObjects(t *testing.T) {
expectedResp: []*models.BucketObject{
&models.BucketObject{
Name: "obj1",
LastModified: t1.String(),
LastModified: t1.Format(time.RFC3339),
Size: int64(1024),
ContentType: "content",
},
@@ -453,12 +453,12 @@ func Test_listObjects(t *testing.T) {
expectedResp: []*models.BucketObject{
&models.BucketObject{
Name: "obj1",
LastModified: t1.String(),
LastModified: t1.Format(time.RFC3339),
Size: int64(1024),
ContentType: "content",
}, &models.BucketObject{
Name: "obj2",
LastModified: t1.String(),
LastModified: t1.Format(time.RFC3339),
Size: int64(512),
ContentType: "content",
},
@@ -520,12 +520,12 @@ func Test_listObjects(t *testing.T) {
expectedResp: []*models.BucketObject{
&models.BucketObject{
Name: "obj1",
LastModified: t1.String(),
LastModified: t1.Format(time.RFC3339),
Size: int64(1024),
ContentType: "content",
}, &models.BucketObject{
Name: "obj2",
LastModified: t1.String(),
LastModified: t1.Format(time.RFC3339),
Size: int64(512),
ContentType: "content",
},