mirrors/object-browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add mcs admin trace api (#82)

Browse Source 
Trace Api uses websocket to send trace information, a
valid jwt token needs to be sent either on the header
or as a cookie of the ws request to start.
Three goroutines are needed to ensure communication
if read hearbeat fails all trace should stop by cancelling
the context. WaitGroups are needed to ensure all
goroutines finish gracefully.
This commit is contained in:
César Nieto
2020-04-30 10:50:51 -07:00
committed by GitHub
parent 9df9309c66
commit 8e9bd8728a
17 changed files with 732 additions and 408 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
go.mod
View File

@@ -12,6 +12,7 @@ require (
github.com/go-openapi/strfmt v0.19.5
github.com/go-openapi/swag v0.19.8
github.com/go-openapi/validate v0.19.7
github.com/gorilla/websocket v1.4.2
github.com/jessevdk/go-flags v1.4.0
github.com/json-iterator/go v1.1.9
github.com/minio/cli v1.22.0

4
go.sum
View File

@@ -42,6 +42,7 @@ github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a/go.mod h1:l
github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496 h1:zV3ejI06GQ59hwDQAvmK1qxOQGB3WuVTRoY0okPTAv0=
github.com/asaskevich/govalidator v0.0.0-20200108200545-475eaeb16496/go.mod h1:oGkLhpf+kjZl6xBf758TQhh5XrAeiJv/7FRz/2spLIg=
github.com/aws/aws-sdk-go v1.20.21/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f h1:ZNv7On9kyUzm7fvRZumSyy/IUiSC7AzL0I1jKKtwooA=
github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc=
github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2 h1:M+TYzBcNIRyzPRg66ndEqUMd7oWDmhvdQmaPC6EZNwM=
github.com/bcicen/jstream v0.0.0-20190220045926-16c1f8af81c2/go.mod h1:RDu/qcrnpEdJC/p8tx34+YBFqqX71lB7dOX9QE+ZC4M=
@@ -254,6 +255,8 @@ github.com/gorilla/rpc v1.2.0 h1:WvvdC2lNeT1SP32zrIce5l0ECBfbAlmrmSBsuc57wfk=
github.com/gorilla/rpc v1.2.0/go.mod h1:V4h9r+4sF5HnzqbwIez0fKSpANP0zlYd3qR7p36jkTQ=
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg=
github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE=
@@ -351,6 +354,7 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kurin/blazer v0.5.4-0.20200327014341-8f90a40f8af7 h1:smZXPopqRVVywwzou4WYWvUbJvSAzIDFizfWElpmAqY=
github.com/kurin/blazer v0.5.4-0.20200327014341-8f90a40f8af7/go.mod h1:4FCXMUWo9DllR2Do4TtBd377ezyAJ51vB5uTBjt0pGU=
github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4=
github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=

56
pkg/ws/websocket.go Normal file
View File

@@ -0,0 +1,56 @@
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
// Package ws contains websocket utils for mcs project
package ws
import (
"net/http"
"strings"
"github.com/go-openapi/errors"
"github.com/minio/mcs/pkg/auth"
)
// Authenticate validates websocket header and returns mcs jwt claims
//
// Authorization Header needs to be like "Authorization Bearer <jwt_token>"
func Authenticate(r *http.Request) (*auth.DecryptedClaims, error) {
// Get Auth token
var reqToken string
// Token might come either as a Cookie or as a Header
// if not set in cookie, check if it is set on Header.
tokenCookie, err := r.Cookie("token")
if err != nil {
headerToken := r.Header.Get("Authorization")
// reqToken should come as "Bearer <token>"
splitHeaderToken := strings.Split(headerToken, "Bearer")
if len(splitHeaderToken) <= 1 {
return nil, errors.New(http.StatusBadRequest, "Authentication not valid")
}
reqToken = strings.TrimSpace(splitHeaderToken[1])
} else {
reqToken = strings.TrimSpace(tokenCookie.Value)
}
// Perform authentication before upgrading to a Websocket Connection
claims, err := auth.JWTAuthenticate(reqToken)
if err != nil {
return nil, errors.New(http.StatusUnauthorized, err.Error())
}
return claims, nil
}

196
restapi/admin_trace.go Normal file
View File

@@ -0,0 +1,196 @@
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package restapi
import (
"context"
"encoding/json"
"fmt"
"log"
"net"
"net/http"
"strings"
"sync"
"github.com/gorilla/websocket"
"github.com/minio/minio/pkg/madmin"
)
// shortTraceMsg Short trace record
type shortTraceMsg struct {
Host string `json:"host"`
Time string `json:"time"`
Client string `json:"client"`
CallStats callStats `json:"callStats"`
FuncName string `json:"api"`
Path string `json:"path"`
Query string `json:"query"`
StatusCode int `json:"statusCode"`
StatusMsg string `json:"statusMsg"`
}
type callStats struct {
Rx int `json:"rx"`
Tx int `json:"tx"`
Duration string `json:"duration"`
Ttfb string `json:"timeToFirstByte"`
}
// trace serves madmin.ServiceTraceInfo
// on a Websocket connection.
func (wsc *wsClient) trace() {
defer func() {
log.Println("trace stopped")
// close connection after return
wsc.conn.close()
}()
log.Println("trace started")
err := startTraceInfo(wsc.conn, wsc.madmin)
// Send Connection Close Message indicating the Status Code
// see https://tools.ietf.org/html/rfc6455#page-45
if err != nil {
// If connection exceeded read deadline send Close
// Message Policy Violation code since we don't want
// to let the receiver figure out the read deadline.
// This is a generic code designed if there is a
// need to hide specific details about the policy.
if nErr, ok := err.(net.Error); ok && nErr.Timeout() {
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.ClosePolicyViolation, ""))
return
}
// else, internal server error
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, err.Error()))
return
}
// normal closure
wsc.conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
}
// startTraceInfo starts trace of the servers
// by first setting a websocket reader that will
// check for a heartbeat.
//
// A WaitGroup is used to handle goroutines and to ensure
// all finish in the proper order. If any, sendTraceInfo()
// or wsReadCheck() returns, trace should end.
func startTraceInfo(conn WSConn, client MinioAdmin) (mError error) {
// a WaitGroup waits for a collection of goroutines to finish
wg := sync.WaitGroup{}
// a cancel context is needed to end all goroutines used
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
// Set number of goroutines to wait. wg.Wait()
// waitsuntil counter is zero (all are done)
wg.Add(3)
// start go routine for reading websocket heartbeat
readErr := wsReadCheck(ctx, &wg, conn)
// send Stream of Trace Info to the ws c.connection
traceCh := sendTraceInfo(ctx, &wg, conn, client)
// If wsReadCheck returns it means that it is not possible to check
// ws heartbeat anymore so we stop from doing trace, cancel context
// for all goroutines.
go func(wg *sync.WaitGroup) {
defer wg.Done()
if err := <-readErr; err != nil {
log.Println("error on wsReadCheck:", err)
mError = err
}
// cancel context for all goroutines.
cancel()
}(&wg)
// wait for traceCh to finish
if err := <-traceCh; err != nil {
mError = err
}
// if traceCh closes for any reason,
// cancel context for all goroutines
cancel()
// wait all goroutines to finish
wg.Wait()
return mError
}
// sendTraceInfo sends stream of Trace Info to the ws connection
func sendTraceInfo(ctx context.Context, wg *sync.WaitGroup, conn WSConn, client MinioAdmin) <-chan error {
// decrements the WaitGroup counter
// by one when the function returns
defer wg.Done()
ch := make(chan error)
go func(ch chan<- error) {
defer close(ch)
// trace all traffic
allTraffic := true
// Trace failed requests only
errOnly := false
// Start listening on all trace activity.
traceCh := client.serviceTrace(ctx, allTraffic, errOnly)
for traceInfo := range traceCh {
if traceInfo.Err != nil {
log.Println("error on serviceTrace:", traceInfo.Err)
ch <- traceInfo.Err
return
}
// Serialize message to be sent
traceInfoBytes, err := json.Marshal(shortTrace(&traceInfo))
if err != nil {
fmt.Println("error on json.Marshal:", err)
ch <- err
return
}
// Send Message through websocket connection
err = conn.writeMessage(websocket.TextMessage, traceInfoBytes)
if err != nil {
log.Println("error writeMessage:", err)
ch <- err
return
}
}
// TODO: verbose
}(ch)
return ch
}
// shortTrace creates a shorter Trace Info message.
// Same implementation as github/minio/mc/cmd/admin-trace.go
func shortTrace(info *madmin.ServiceTraceInfo) shortTraceMsg {
t := info.Trace
s := shortTraceMsg{}
s.Time = t.ReqInfo.Time.String()
s.Path = t.ReqInfo.Path
s.Query = t.ReqInfo.RawQuery
s.FuncName = t.FuncName
s.StatusCode = t.RespInfo.StatusCode
s.StatusMsg = http.StatusText(t.RespInfo.StatusCode)
s.CallStats.Duration = t.CallStats.Latency.String()
s.CallStats.Rx = t.CallStats.InputBytes
s.CallStats.Tx = t.CallStats.OutputBytes
if host, ok := t.ReqInfo.Headers["Host"]; ok {
s.Host = strings.Join(host, "")
}
cSlice := strings.Split(t.ReqInfo.Client, ":")
s.Client = cSlice[0]
return s
}

169
restapi/admin_trace_test.go Normal file
View File

@@ -0,0 +1,169 @@
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package restapi
import (
"context"
"encoding/json"
"fmt"
"testing"
"github.com/gorilla/websocket"
"github.com/minio/minio/pkg/madmin"
trace "github.com/minio/minio/pkg/trace"
"github.com/stretchr/testify/assert"
)
// assigning mock at runtime instead of compile time
var minioServiceTraceMock func(ctx context.Context, allTrace, errTrace bool) <-chan madmin.ServiceTraceInfo
// mock function of listPolicies()
func (ac adminClientMock) serviceTrace(ctx context.Context, allTrace, errTrace bool) <-chan madmin.ServiceTraceInfo {
return minioServiceTraceMock(ctx, allTrace, errTrace)
}
func TestAdminTrace(t *testing.T) {
assert := assert.New(t)
adminClient := adminClientMock{}
mockWSConn := mockConn{}
wsClientMock := wsClientMock{madmin: adminClient}
function := "startTraceInfo()"
testReceiver := make(chan shortTraceMsg, 5)
textToReceive := "test"
testStreamSize := 5
// Test-1: Serve Trace with no errors until trace finishes sending
// define mock function behavior for minio server Trace
minioServiceTraceMock = func(ctx context.Context, allTrace, errTrace bool) <-chan madmin.ServiceTraceInfo {
ch := make(chan madmin.ServiceTraceInfo)
// Only success, start a routine to start reading line by line.
go func(ch chan<- madmin.ServiceTraceInfo) {
defer close(ch)
lines := make([]int, testStreamSize)
// mocking sending 5 lines of info
for range lines {
info := trace.Info{
FuncName: textToReceive,
}
ch <- madmin.ServiceTraceInfo{Trace: info}
}
}(ch)
return ch
}
// mock function of conn.ReadMessage(), no error on read
connReadMessageMock = func() (messageType int, p []byte, err error) {
return 0, []byte{}, nil
}
writesCount := 1
// mock connection WriteMessage() no error
connWriteMessageMock = func(messageType int, data []byte) error {
// emulate that receiver gets the message written
var t shortTraceMsg
_ = json.Unmarshal(data, &t)
if writesCount == testStreamSize {
// for testing we need to close the receiver channel
close(testReceiver)
return nil
}
testReceiver <- t
writesCount++
return nil
}
if err := startTraceInfo(mockWSConn, wsClientMock.madmin); err != nil {
t.Errorf("Failed on %s:, error occurred: %s", function, err.Error())
}
// check that the TestReceiver got the same number of data from trace.
for i := range testReceiver {
assert.Equal(textToReceive, i.FuncName)
}
// Test-2: if error happens while writing, return error
connWriteMessageMock = func(messageType int, data []byte) error {
return fmt.Errorf("error on write")
}
if err := startTraceInfo(mockWSConn, wsClientMock.madmin); assert.Error(err) {
assert.Equal("error on write", err.Error())
}
// Test-3: error happens while reading, unexpected Close Error should return error.
connWriteMessageMock = func(messageType int, data []byte) error {
return nil
}
// mock function of conn.ReadMessage(), returns unexpected Close Error CloseAbnormalClosure
connReadMessageMock = func() (messageType int, p []byte, err error) {
return 0, []byte{}, &websocket.CloseError{Code: websocket.CloseAbnormalClosure, Text: ""}
}
if err := startTraceInfo(mockWSConn, wsClientMock.madmin); assert.Error(err) {
assert.Equal("websocket: close 1006 (abnormal closure)", err.Error())
}
// Test-4: error happens while reading, expected Close Error NormalClosure
// expected Close Error should not return an error, just end trace.
connReadMessageMock = func() (messageType int, p []byte, err error) {
return 0, []byte{}, &websocket.CloseError{Code: websocket.CloseNormalClosure, Text: ""}
}
if err := startTraceInfo(mockWSConn, wsClientMock.madmin); err != nil {
t.Errorf("Failed on %s:, error occurred: %s", function, err.Error())
}
// Test-5: error happens while reading, expected Close Error CloseGoingAway
// expected Close Error should not return an error, just return.
connReadMessageMock = func() (messageType int, p []byte, err error) {
return 0, []byte{}, &websocket.CloseError{Code: websocket.CloseGoingAway, Text: ""}
}
if err := startTraceInfo(mockWSConn, wsClientMock.madmin); err != nil {
t.Errorf("Failed on %s:, error occurred: %s", function, err.Error())
}
// Test-6: error happens while reading, non Close Error Type should be returned as
// error
connReadMessageMock = func() (messageType int, p []byte, err error) {
return 0, []byte{}, fmt.Errorf("error on read")
}
if err := startTraceInfo(mockWSConn, wsClientMock.madmin); assert.Error(err) {
assert.Equal("error on read", err.Error())
}
// Test-7: error happens on serviceTrace Minio, trace should stop
// and error shall be returned.
minioServiceTraceMock = func(ctx context.Context, allTrace, errTrace bool) <-chan madmin.ServiceTraceInfo {
ch := make(chan madmin.ServiceTraceInfo)
// Only success, start a routine to start reading line by line.
go func(ch chan<- madmin.ServiceTraceInfo) {
defer close(ch)
lines := make([]int, 2)
// mocking sending 5 lines of info
for range lines {
info := trace.Info{
NodeName: "test",
}
ch <- madmin.ServiceTraceInfo{Trace: info}
}
ch <- madmin.ServiceTraceInfo{Err: fmt.Errorf("error on trace")}
}(ch)
return ch
}
// mock function of conn.ReadMessage(), no error on read, should stay unless
// context is done.
connReadMessageMock = func() (messageType int, p []byte, err error) {
return 0, []byte{}, nil
}
if err := startTraceInfo(mockWSConn, wsClientMock.madmin); assert.Error(err) {
assert.Equal("error on trace", err.Error())
}
}

19
restapi/client-admin.go
View File

@@ -55,7 +55,7 @@ func NewAdminClient(url, accessKey, secretKey string) (*madmin.AdminClient, *pro
// it also enables an internal trace transport.
var s3AdminNew = mcCmd.NewAdminFactory()
// Define MinioAdmin interface with all functions to be implemented
// MinioAdmin interface with all functions to be implemented
// by mock when testing, it should include all MinioAdmin respective api calls
// that are used within this project.
type MinioAdmin interface {
@@ -80,6 +80,7 @@ type MinioAdmin interface {
serverInfo(ctx context.Context) (madmin.InfoMessage, error)
startProfiling(ctx context.Context, profiler madmin.ProfilerType) ([]madmin.StartProfilingResult, error)
stopProfiling(ctx context.Context) (io.ReadCloser, error)
serviceTrace(ctx context.Context, allTrace, errTrace bool) <-chan madmin.ServiceTraceInfo
// Service Accounts
addServiceAccount(ctx context.Context, policy *iampolicy.Policy) (mauth.Credentials, error)
}
@@ -197,6 +198,11 @@ func (ac adminClient) stopProfiling(ctx context.Context) (io.ReadCloser, error)
return ac.client.DownloadProfilingData(ctx)
}
// implements madmin.ServiceTrace()
func (ac adminClient) serviceTrace(ctx context.Context, allTrace, errTrace bool) <-chan madmin.ServiceTraceInfo {
return ac.client.ServiceTrace(ctx, allTrace, errTrace)
}
// implements madmin.AddServiceAccount()
func (ac adminClient) addServiceAccount(ctx context.Context, policy *iampolicy.Policy) (mauth.Credentials, error) {
return ac.client.AddServiceAccount(ctx, policy)
@@ -216,3 +222,14 @@ func newMAdminClient(jwt string) (*madmin.AdminClient, error) {
}
return adminClient, nil
}
func newSuperMAdminClient() (*madmin.AdminClient, error) {
endpoint := getMinIOServer()
accessKeyID := getAccessKey()
secretAccessKey := getSecretKey()
adminClient, pErr := NewAdminClient(endpoint, accessKeyID, secretAccessKey)
if pErr != nil {
return nil, pErr.Cause
}
return adminClient, nil
}

4
restapi/client.go
View File

@@ -34,7 +34,7 @@ func init() {
minio.MaxRetry = 1
}
// Define MinioClient interface with all functions to be implemented
// MinioClient interface with all functions to be implemented
// by mock when testing, it should include all MinioClient respective api calls
// that are used within this project.
type MinioClient interface {
@@ -84,7 +84,7 @@ func (c minioClient) getBucketPolicy(bucketName string) (string, error) {
return c.client.GetBucketPolicy(bucketName)
}
// Define MCS3Client interface with all functions to be implemented
// MCS3Client interface with all functions to be implemented
// by mock when testing, it should include all mc/S3Client respective api calls
// that are used within this project.
type MCS3Client interface {

7
restapi/configure_mcs.go
View File

@@ -161,9 +161,12 @@ func setupGlobalMiddleware(handler http.Handler) http.Handler {
// FileServerMiddleware serves files from the static folder
func FileServerMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.HasPrefix(r.URL.Path, "/api") {
switch {
case strings.HasPrefix(r.URL.Path, "/ws"):
serveWS(w, r)
case strings.HasPrefix(r.URL.Path, "/api"):
next.ServeHTTP(w, r)
} else {
default:
http.FileServer(&assetfs.AssetFS{
Asset: portalUI.Asset,
AssetDir: portalUI.AssetDir,

1
restapi/doc.go
View File

@@ -20,6 +20,7 @@
//
// Schemes:
// http
// ws
// Host: localhost
// BasePath: /api/v1
// Version: 0.1.0

6
restapi/embedded_spec.go
View File

@@ -42,7 +42,8 @@ func init() {
"application/json"
],
"schemes": [
"http"
"http",
"ws"
],
"swagger": "2.0",
"info": {
@@ -2032,7 +2033,8 @@ func init() {
"application/json"
],
"schemes": [
"http"
"http",
"ws"
],
"swagger": "2.0",
"info": {

90
restapi/operations/bulk_update_users_groups.go
View File

@@ -1,90 +0,0 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package operations
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the generate command
import (
"net/http"
"github.com/go-openapi/runtime/middleware"
"github.com/minio/mcs/models"
)
// BulkUpdateUsersGroupsHandlerFunc turns a function with the right signature into a bulk update users groups handler
type BulkUpdateUsersGroupsHandlerFunc func(BulkUpdateUsersGroupsParams, *models.Principal) middleware.Responder
// Handle executing the request and returning a response
func (fn BulkUpdateUsersGroupsHandlerFunc) Handle(params BulkUpdateUsersGroupsParams, principal *models.Principal) middleware.Responder {
return fn(params, principal)
}
// BulkUpdateUsersGroupsHandler interface for that can handle valid bulk update users groups params
type BulkUpdateUsersGroupsHandler interface {
Handle(BulkUpdateUsersGroupsParams, *models.Principal) middleware.Responder
}
// NewBulkUpdateUsersGroups creates a new http.Handler for the bulk update users groups operation
func NewBulkUpdateUsersGroups(ctx *middleware.Context, handler BulkUpdateUsersGroupsHandler) *BulkUpdateUsersGroups {
return &BulkUpdateUsersGroups{Context: ctx, Handler: handler}
}
/*BulkUpdateUsersGroups swagger:route PUT /user/groups bulkUpdateUsersGroups
Bulk functionality to Add Users to Groups
*/
type BulkUpdateUsersGroups struct {
Context *middleware.Context
Handler BulkUpdateUsersGroupsHandler
}
func (o *BulkUpdateUsersGroups) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
route, rCtx, _ := o.Context.RouteInfo(r)
if rCtx != nil {
r = rCtx
}
var Params = NewBulkUpdateUsersGroupsParams()
uprinc, aCtx, err := o.Context.Authorize(r, route)
if err != nil {
o.Context.Respond(rw, r, route.Produces, route, err)
return
}
if aCtx != nil {
r = aCtx
}
var principal *models.Principal
if uprinc != nil {
principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise
}
if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params
o.Context.Respond(rw, r, route.Produces, route, err)
return
}
res := o.Handler.Handle(Params, principal) // actually handle the request
o.Context.Respond(rw, r, route.Produces, route, res)
}

94
restapi/operations/bulk_update_users_groups_parameters.go
View File

@@ -1,94 +0,0 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package operations
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the swagger generate command
import (
"io"
"net/http"
"github.com/go-openapi/errors"
"github.com/go-openapi/runtime"
"github.com/go-openapi/runtime/middleware"
"github.com/minio/mcs/models"
)
// NewBulkUpdateUsersGroupsParams creates a new BulkUpdateUsersGroupsParams object
// no default values defined in spec.
func NewBulkUpdateUsersGroupsParams() BulkUpdateUsersGroupsParams {
return BulkUpdateUsersGroupsParams{}
}
// BulkUpdateUsersGroupsParams contains all the bound params for the bulk update users groups operation
// typically these are obtained from a http.Request
//
// swagger:parameters BulkUpdateUsersGroups
type BulkUpdateUsersGroupsParams struct {
// HTTP Request Object
HTTPRequest *http.Request `json:"-"`
/*
Required: true
In: body
*/
Body *models.BulkUserGroups
}
// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface
// for simple values it will use straight method calls.
//
// To ensure default values, the struct must have been initialized with NewBulkUpdateUsersGroupsParams() beforehand.
func (o *BulkUpdateUsersGroupsParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error {
var res []error
o.HTTPRequest = r
if runtime.HasBody(r) {
defer r.Body.Close()
var body models.BulkUserGroups
if err := route.Consumer.Consume(r.Body, &body); err != nil {
if err == io.EOF {
res = append(res, errors.Required("body", "body"))
} else {
res = append(res, errors.NewParseError("body", "body", "", err))
}
} else {
// validate body object
if err := body.Validate(route.Formats); err != nil {
res = append(res, err)
}
if len(res) == 0 {
o.Body = &body
}
}
} else {
res = append(res, errors.Required("body", "body"))
}
if len(res) > 0 {
return errors.CompositeValidationError(res...)
}
return nil
}

113
restapi/operations/bulk_update_users_groups_responses.go
View File

@@ -1,113 +0,0 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package operations
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the swagger generate command
import (
"net/http"
"github.com/go-openapi/runtime"
"github.com/minio/mcs/models"
)
// BulkUpdateUsersGroupsOKCode is the HTTP code returned for type BulkUpdateUsersGroupsOK
const BulkUpdateUsersGroupsOKCode int = 200
/*BulkUpdateUsersGroupsOK A successful response.
swagger:response bulkUpdateUsersGroupsOK
*/
type BulkUpdateUsersGroupsOK struct {
}
// NewBulkUpdateUsersGroupsOK creates BulkUpdateUsersGroupsOK with default headers values
func NewBulkUpdateUsersGroupsOK() *BulkUpdateUsersGroupsOK {
return &BulkUpdateUsersGroupsOK{}
}
// WriteResponse to the client
func (o *BulkUpdateUsersGroupsOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) {
rw.Header().Del(runtime.HeaderContentType) //Remove Content-Type on empty responses
rw.WriteHeader(200)
}
/*BulkUpdateUsersGroupsDefault Generic error response.
swagger:response bulkUpdateUsersGroupsDefault
*/
type BulkUpdateUsersGroupsDefault struct {
_statusCode int
/*
In: Body
*/
Payload *models.Error `json:"body,omitempty"`
}
// NewBulkUpdateUsersGroupsDefault creates BulkUpdateUsersGroupsDefault with default headers values
func NewBulkUpdateUsersGroupsDefault(code int) *BulkUpdateUsersGroupsDefault {
if code <= 0 {
code = 500
}
return &BulkUpdateUsersGroupsDefault{
_statusCode: code,
}
}
// WithStatusCode adds the status to the bulk update users groups default response
func (o *BulkUpdateUsersGroupsDefault) WithStatusCode(code int) *BulkUpdateUsersGroupsDefault {
o._statusCode = code
return o
}
// SetStatusCode sets the status to the bulk update users groups default response
func (o *BulkUpdateUsersGroupsDefault) SetStatusCode(code int) {
o._statusCode = code
}
// WithPayload adds the payload to the bulk update users groups default response
func (o *BulkUpdateUsersGroupsDefault) WithPayload(payload *models.Error) *BulkUpdateUsersGroupsDefault {
o.Payload = payload
return o
}
// SetPayload sets the payload to the bulk update users groups default response
func (o *BulkUpdateUsersGroupsDefault) SetPayload(payload *models.Error) {
o.Payload = payload
}
// WriteResponse to the client
func (o *BulkUpdateUsersGroupsDefault) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) {
rw.WriteHeader(o._statusCode)
if o.Payload != nil {
payload := o.Payload
if err := producer.Produce(rw, payload); err != nil {
panic(err) // let the recovery middleware deal with this
}
}
}

104
restapi/operations/bulk_update_users_groups_urlbuilder.go
View File

@@ -1,104 +0,0 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package operations
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the generate command
import (
"errors"
"net/url"
golangswaggerpaths "path"
)
// BulkUpdateUsersGroupsURL generates an URL for the bulk update users groups operation
type BulkUpdateUsersGroupsURL struct {
_basePath string
}
// WithBasePath sets the base path for this url builder, only required when it's different from the
// base path specified in the swagger spec.
// When the value of the base path is an empty string
func (o *BulkUpdateUsersGroupsURL) WithBasePath(bp string) *BulkUpdateUsersGroupsURL {
o.SetBasePath(bp)
return o
}
// SetBasePath sets the base path for this url builder, only required when it's different from the
// base path specified in the swagger spec.
// When the value of the base path is an empty string
func (o *BulkUpdateUsersGroupsURL) SetBasePath(bp string) {
o._basePath = bp
}
// Build a url path and query string
func (o *BulkUpdateUsersGroupsURL) Build() (*url.URL, error) {
var _result url.URL
var _path = "/user/groups"
_basePath := o._basePath
if _basePath == "" {
_basePath = "/api/v1"
}
_result.Path = golangswaggerpaths.Join(_basePath, _path)
return &_result, nil
}
// Must is a helper function to panic when the url builder returns an error
func (o *BulkUpdateUsersGroupsURL) Must(u *url.URL, err error) *url.URL {
if err != nil {
panic(err)
}
if u == nil {
panic("url can't be nil")
}
return u
}
// String returns the string representation of the path with query string
func (o *BulkUpdateUsersGroupsURL) String() string {
return o.Must(o.Build()).String()
}
// BuildFull builds a full url with scheme, host, path and query string
func (o *BulkUpdateUsersGroupsURL) BuildFull(scheme, host string) (*url.URL, error) {
if scheme == "" {
return nil, errors.New("scheme is required for a full url on BulkUpdateUsersGroupsURL")
}
if host == "" {
return nil, errors.New("host is required for a full url on BulkUpdateUsersGroupsURL")
}
base, err := o.Build()
if err != nil {
return nil, err
}
base.Scheme = scheme
base.Host = host
return base, nil
}
// StringFull returns the string representation of a complete url
func (o *BulkUpdateUsersGroupsURL) StringFull(scheme, host string) string {
return o.Must(o.BuildFull(scheme, host)).String()
}

207
restapi/ws_handle.go Normal file
View File

@@ -0,0 +1,207 @@
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package restapi
import (
"context"
"log"
"net/http"
"strings"
"sync"
"time"
"github.com/go-openapi/errors"
"github.com/gorilla/websocket"
"github.com/minio/mcs/pkg/ws"
)
var upgrader = websocket.Upgrader{
ReadBufferSize: 0,
WriteBufferSize: 1024,
}
const (
// websocket base path
wsBasePath = "/ws"
// Time allowed to read the next pong message from the peer.
pingWait = 15 * time.Second
// Maximum message size allowed from peer. 0 = unlimited
maxMessageSize = 512
)
// MCSWebsocket interface of a Websocket Client
type MCSWebsocket interface {
// start trace info from servers
trace()
}
type wsClient struct {
// websocket connection.
conn wsConn
// MinIO admin Client
madmin MinioAdmin
}
// WSConn interface with all functions to be implemented
// by mock when testing, it should include all websocket.Conn
// respective api calls that are used within this project.
type WSConn interface {
writeMessage(messageType int, data []byte) error
close() error
setReadLimit(limit int64)
setReadDeadline(t time.Time) error
setPongHandler(h func(appData string) error)
readMessage() (messageType int, p []byte, err error)
}
// Interface implementation
//
// Define the structure of a websocket Connection
type wsConn struct {
conn *websocket.Conn
}
func (c wsConn) writeMessage(messageType int, data []byte) error {
return c.conn.WriteMessage(messageType, data)
}
func (c wsConn) close() error {
return c.conn.Close()
}
func (c wsConn) setReadLimit(limit int64) {
c.conn.SetReadLimit(limit)
}
func (c wsConn) setReadDeadline(t time.Time) error {
return c.conn.SetReadDeadline(t)
}
func (c wsConn) setPongHandler(h func(appData string) error) {
c.conn.SetPongHandler(h)
}
func (c wsConn) readMessage() (messageType int, p []byte, err error) {
return c.conn.ReadMessage()
}
// serveWS validates the incoming request and
// upgrades the request to a Websocket protocol.
// Websocket communication will be done depending
// on the path.
// Request should come like ws://<host>:<port>/ws/<api>
//
// TODO: Enable CORS
func serveWS(w http.ResponseWriter, req *http.Request) {
// authenticate WS connection
// TODO: use this claims to create the adminClient
_, err := ws.Authenticate(req)
if err != nil {
log.Print("error on ws authentication: ", err)
errors.ServeError(w, req, err)
return
}
// upgrades the HTTP server connection to the WebSocket protocol.
conn, err := upgrader.Upgrade(w, req, nil)
if err != nil {
log.Print("error on upgrade: ", err)
errors.ServeError(w, req, err)
return
}
// TODO: CHANGE ! to use newMAdminClient once Assume Role is
// allowed to do Trace use jwt on ws.
// Using newSuperMAdminClient in the meantime for sake of functionality
// Only start Websocket Interaction after user has been
// authenticated.
mAdmin, err := newSuperMAdminClient()
if err != nil {
log.Println("error creating Madmin Client:", err)
errors.ServeError(w, req, err)
return
}
// create a minioClient interface implementation
// defining the client to be used
adminClient := adminClient{client: mAdmin}
// create a websocket connection interface implementation
// defining the connection to be used
wsConnection := wsConn{conn: conn}
// create websocket client and handle request
wsClient := &wsClient{conn: wsConnection, madmin: adminClient}
switch strings.TrimPrefix(req.URL.Path, wsBasePath) {
case "/trace":
go wsClient.trace()
default:
// path not found
conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
conn.Close()
}
}
// wsReadCheck ensures that the client is sending a message
// every `pingWait` seconds. If deadline exceeded or an error
// happened this will return, meaning it won't be able to ensure
// client heartbeat.
func wsReadCheck(ctx context.Context, wg *sync.WaitGroup, conn WSConn) chan error {
// decrements the WaitGroup counter by one when the function returns
defer wg.Done()
ch := make(chan error)
go func(ch chan error) {
defer close(ch)
// set initial Limits and deadlines for the Reader
conn.setReadLimit(maxMessageSize)
conn.setReadDeadline(time.Now().Add(pingWait))
conn.setPongHandler(func(string) error { conn.setReadDeadline(time.Now().Add(pingWait)); return nil })
for {
select {
case <-ctx.Done():
log.Println("context done inside wsReadCheck")
return
default:
_, _, err := conn.readMessage()
if err != nil {
// if error of type websocket.CloseError and is Unexpected
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseNormalClosure) {
log.Println("error unexpected CloseError on ReadMessage:", err)
ch <- err
return
}
// Not all errors are of type websocket.CloseError.
// If not of type websocket.CloseError return error
if _, ok := err.(*websocket.CloseError); !ok {
log.Println("error on ReadMessage:", err)
ch <- err
return
}
// else is an expected Close Error
log.Println("closed conn.ReadMessage:", err)
return
}
// Reset Read Deadline after each Read
conn.setReadDeadline(time.Now().Add(pingWait))
conn.setPongHandler(func(string) error { conn.setReadDeadline(time.Now().Add(pingWait)); return nil })
}
}
}(ch)
return ch
}

68
restapi/ws_handle_test.go Normal file
View File

@@ -0,0 +1,68 @@
// This file is part of MinIO Console Server
// Copyright (c) 2020 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package restapi
import (
"time"
)
// Common mocks for WSConn interface
// assigning mock at runtime instead of compile time
var connWriteMessageMock func(messageType int, data []byte) error
var connReadMessageMock func() (messageType int, p []byte, err error)
// Uncomment and implement if needed
// var connCloseMock func() error
// var connSetReadLimitMock func(limit int64)
// var connSetReadDeadlineMock func(t time.Time) error
// var connSetPongHandlerMock func(h func(appData string) error)
// The Conn type represents a WebSocket connection.
type mockConn struct{}
func (c mockConn) writeMessage(messageType int, data []byte) error {
return connWriteMessageMock(messageType, data)
}
func (c mockConn) readMessage() (messageType int, p []byte, err error) {
return connReadMessageMock()
}
func (c mockConn) close() error {
return nil
}
func (c mockConn) setReadLimit(limit int64) {
}
func (c mockConn) setReadDeadline(t time.Time) error {
return nil
}
func (c mockConn) setPongHandler(h func(appData string) error) {
}
// Common mocks for MCSWebsocket interface
// assigning mock at runtime instead of compile time
var wsTraceMock func()
// Define a mock struct of wsClient interface implementation
type wsClientMock struct {
// MinIO admin Client
madmin MinioAdmin
}
// mock function of wsc.trace()
func (wsc wsClientMock) trace() {
wsTraceMock()
}

1
swagger.yml
View File

@@ -8,6 +8,7 @@ produces:
- application/json
schemes:
- http
- ws
basePath: /api/v1
# We are going to be taking `Authorization: Bearer TOKEN` header for our authentication
securityDefinitions: