remove gateway completely (#15929)
This commit is contained in:
Harshavardhana
GitHub
@@ -18,12 +18,9 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/minio/minio/internal/sync/errgroup"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -58,13 +55,6 @@ var globalObjectAPI ObjectLayer
|
||||
// Global cacheObjects, only accessed by newCacheObjectsFn().
|
||||
var globalCacheObjectAPI CacheObjectLayer
|
||||
|
||||
// Checks if the object is a directory, this logic uses
|
||||
// if size == 0 and object ends with SlashSeparator then
|
||||
// returns true.
|
||||
func isObjectDir(object string, size int64) bool {
|
||||
return HasSuffix(object, SlashSeparator) && size == 0
|
||||
}
|
||||
|
||||
func newStorageAPIWithoutHealthCheck(endpoint Endpoint) (storage StorageAPI, err error) {
|
||||
if endpoint.IsLocal {
|
||||
storage, err := newXLStorage(endpoint)
|
||||
@@ -89,326 +79,3 @@ func newStorageAPI(endpoint Endpoint) (storage StorageAPI, err error) {
|
||||
|
||||
return newStorageRESTClient(endpoint, true), nil
|
||||
}
|
||||
|
||||
func listObjectsNonSlash(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
|
||||
endWalkCh := make(chan struct{})
|
||||
defer close(endWalkCh)
|
||||
recursive := true
|
||||
walkResultCh := startTreeWalk(ctx, bucket, prefix, "", recursive, listDir, isLeaf, isLeafDir, endWalkCh)
|
||||
|
||||
var objInfos []ObjectInfo
|
||||
var eof bool
|
||||
var prevPrefix string
|
||||
|
||||
for {
|
||||
if len(objInfos) == maxKeys {
|
||||
break
|
||||
}
|
||||
result, ok := <-walkResultCh
|
||||
if !ok {
|
||||
eof = true
|
||||
break
|
||||
}
|
||||
|
||||
var objInfo ObjectInfo
|
||||
var err error
|
||||
|
||||
index := strings.Index(strings.TrimPrefix(result.entry, prefix), delimiter)
|
||||
if index == -1 {
|
||||
objInfo, err = getObjInfo(ctx, bucket, result.entry)
|
||||
if err != nil {
|
||||
// Ignore errFileNotFound as the object might have got
|
||||
// deleted in the interim period of listing and getObjectInfo(),
|
||||
// ignore quorum error as it might be an entry from an outdated disk.
|
||||
if IsErrIgnored(err, []error{
|
||||
errFileNotFound,
|
||||
errErasureReadQuorum,
|
||||
}...) {
|
||||
continue
|
||||
}
|
||||
return loi, toObjectErr(err, bucket, prefix)
|
||||
}
|
||||
} else {
|
||||
index = len(prefix) + index + len(delimiter)
|
||||
currPrefix := result.entry[:index]
|
||||
if currPrefix == prevPrefix {
|
||||
continue
|
||||
}
|
||||
prevPrefix = currPrefix
|
||||
|
||||
objInfo = ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: currPrefix,
|
||||
IsDir: true,
|
||||
}
|
||||
}
|
||||
|
||||
if objInfo.Name <= marker {
|
||||
continue
|
||||
}
|
||||
|
||||
objInfos = append(objInfos, objInfo)
|
||||
if result.end {
|
||||
eof = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
result := ListObjectsInfo{}
|
||||
for _, objInfo := range objInfos {
|
||||
if objInfo.IsDir {
|
||||
result.Prefixes = append(result.Prefixes, objInfo.Name)
|
||||
continue
|
||||
}
|
||||
result.Objects = append(result.Objects, objInfo)
|
||||
}
|
||||
|
||||
if !eof {
|
||||
result.IsTruncated = true
|
||||
if len(objInfos) > 0 {
|
||||
result.NextMarker = objInfos[len(objInfos)-1].Name
|
||||
} else if len(result.Prefixes) > 0 {
|
||||
result.NextMarker = result.Prefixes[len(result.Prefixes)-1]
|
||||
}
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
// Walk a bucket, optionally prefix recursively, until we have returned
|
||||
// all the content to objectInfo channel, it is callers responsibility
|
||||
// to allocate a receive channel for ObjectInfo, upon any unhandled
|
||||
// error walker returns error. Optionally if context.Done() is received
|
||||
// then Walk() stops the walker.
|
||||
func fsWalk(ctx context.Context, obj ObjectLayer, bucket, prefix string, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc, results chan<- ObjectInfo, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) error {
|
||||
if err := checkListObjsArgs(ctx, bucket, prefix, "", obj); err != nil {
|
||||
// Upon error close the channel.
|
||||
close(results)
|
||||
return err
|
||||
}
|
||||
|
||||
walkResultCh := startTreeWalk(ctx, bucket, prefix, "", true, listDir, isLeaf, isLeafDir, ctx.Done())
|
||||
|
||||
go func() {
|
||||
defer close(results)
|
||||
|
||||
for {
|
||||
walkResult, ok := <-walkResultCh
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
var objInfo ObjectInfo
|
||||
var err error
|
||||
if HasSuffix(walkResult.entry, SlashSeparator) {
|
||||
for _, getObjectInfoDir := range getObjectInfoDirs {
|
||||
objInfo, err = getObjectInfoDir(ctx, bucket, walkResult.entry)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
if err == errFileNotFound {
|
||||
err = nil
|
||||
objInfo = ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: walkResult.entry,
|
||||
IsDir: true,
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
objInfo, err = getObjInfo(ctx, bucket, walkResult.entry)
|
||||
}
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
results <- objInfo
|
||||
if walkResult.end {
|
||||
break
|
||||
}
|
||||
}
|
||||
}()
|
||||
return nil
|
||||
}
|
||||
|
||||
func listObjects(ctx context.Context, obj ObjectLayer, bucket, prefix, marker, delimiter string, maxKeys int, tpool *TreeWalkPool, listDir ListDirFunc, isLeaf IsLeafFunc, isLeafDir IsLeafDirFunc, getObjInfo func(context.Context, string, string) (ObjectInfo, error), getObjectInfoDirs ...func(context.Context, string, string) (ObjectInfo, error)) (loi ListObjectsInfo, err error) {
|
||||
if delimiter != SlashSeparator && delimiter != "" {
|
||||
return listObjectsNonSlash(ctx, bucket, prefix, marker, delimiter, maxKeys, tpool, listDir, isLeaf, isLeafDir, getObjInfo, getObjectInfoDirs...)
|
||||
}
|
||||
|
||||
if err := checkListObjsArgs(ctx, bucket, prefix, marker, obj); err != nil {
|
||||
return loi, err
|
||||
}
|
||||
|
||||
// Marker is set validate pre-condition.
|
||||
if marker != "" {
|
||||
// Marker not common with prefix is not implemented. Send an empty response
|
||||
if !HasPrefix(marker, prefix) {
|
||||
return loi, nil
|
||||
}
|
||||
}
|
||||
|
||||
// With max keys of zero we have reached eof, return right here.
|
||||
if maxKeys == 0 {
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
if len(prefix) > 0 && maxKeys == 1 && delimiter == "" && marker == "" {
|
||||
// Optimization for certain applications like
|
||||
// - Cohesity
|
||||
// - Actifio, Splunk etc.
|
||||
// which send ListObjects requests where the actual object
|
||||
// itself is the prefix and max-keys=1 in such scenarios
|
||||
// we can simply verify locally if such an object exists
|
||||
// to avoid the need for ListObjects().
|
||||
objInfo, err := obj.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
|
||||
if err == nil {
|
||||
loi.Objects = append(loi.Objects, objInfo)
|
||||
return loi, nil
|
||||
}
|
||||
}
|
||||
|
||||
// For delimiter and prefix as '/' we do not list anything at all
|
||||
// since according to s3 spec we stop at the 'delimiter'
|
||||
// along // with the prefix. On a flat namespace with 'prefix'
|
||||
// as '/' we don't have any entries, since all the keys are
|
||||
// of form 'keyName/...'
|
||||
if delimiter == SlashSeparator && prefix == SlashSeparator {
|
||||
return loi, nil
|
||||
}
|
||||
|
||||
// Over flowing count - reset to maxObjectList.
|
||||
if maxKeys < 0 || maxKeys > maxObjectList {
|
||||
maxKeys = maxObjectList
|
||||
}
|
||||
|
||||
// Default is recursive, if delimiter is set then list non recursive.
|
||||
recursive := true
|
||||
if delimiter == SlashSeparator {
|
||||
recursive = false
|
||||
}
|
||||
|
||||
walkResultCh, endWalkCh := tpool.Release(listParams{bucket, recursive, marker, prefix})
|
||||
if walkResultCh == nil {
|
||||
endWalkCh = make(chan struct{})
|
||||
walkResultCh = startTreeWalk(ctx, bucket, prefix, marker, recursive, listDir, isLeaf, isLeafDir, endWalkCh)
|
||||
}
|
||||
|
||||
var eof bool
|
||||
var nextMarker string
|
||||
|
||||
maxConcurrent := maxKeys / 10
|
||||
if maxConcurrent == 0 {
|
||||
maxConcurrent = maxKeys
|
||||
}
|
||||
|
||||
// List until maxKeys requested.
|
||||
g := errgroup.WithNErrs(maxKeys).WithConcurrency(maxConcurrent)
|
||||
|
||||
objInfoFound := make([]*ObjectInfo, maxKeys)
|
||||
var i int
|
||||
for i = 0; i < maxKeys; i++ {
|
||||
i := i
|
||||
walkResult, ok := <-walkResultCh
|
||||
if !ok {
|
||||
if HasSuffix(prefix, SlashSeparator) {
|
||||
objInfo, err := obj.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true})
|
||||
if err == nil {
|
||||
loi.Objects = append(loi.Objects, objInfo)
|
||||
return loi, nil
|
||||
}
|
||||
}
|
||||
// Closed channel.
|
||||
eof = true
|
||||
break
|
||||
}
|
||||
|
||||
if HasSuffix(walkResult.entry, SlashSeparator) {
|
||||
g.Go(func() error {
|
||||
for _, getObjectInfoDir := range getObjectInfoDirs {
|
||||
objInfo, err := getObjectInfoDir(ctx, bucket, walkResult.entry)
|
||||
if err == nil {
|
||||
objInfoFound[i] = &objInfo
|
||||
// Done...
|
||||
return nil
|
||||
}
|
||||
|
||||
// Add temp, may be overridden,
|
||||
if err == errFileNotFound {
|
||||
objInfoFound[i] = &ObjectInfo{
|
||||
Bucket: bucket,
|
||||
Name: walkResult.entry,
|
||||
IsDir: true,
|
||||
}
|
||||
continue
|
||||
}
|
||||
return toObjectErr(err, bucket, prefix)
|
||||
}
|
||||
return nil
|
||||
}, i)
|
||||
} else {
|
||||
g.Go(func() error {
|
||||
objInfo, err := getObjInfo(ctx, bucket, walkResult.entry)
|
||||
if err != nil {
|
||||
// Ignore errFileNotFound as the object might have got
|
||||
// deleted in the interim period of listing and getObjectInfo(),
|
||||
// ignore quorum error as it might be an entry from an outdated disk.
|
||||
if IsErrIgnored(err, []error{
|
||||
errFileNotFound,
|
||||
errErasureReadQuorum,
|
||||
}...) {
|
||||
return nil
|
||||
}
|
||||
return toObjectErr(err, bucket, prefix)
|
||||
}
|
||||
objInfoFound[i] = &objInfo
|
||||
return nil
|
||||
}, i)
|
||||
}
|
||||
|
||||
if walkResult.end {
|
||||
eof = true
|
||||
break
|
||||
}
|
||||
}
|
||||
for _, err := range g.Wait() {
|
||||
if err != nil {
|
||||
return loi, err
|
||||
}
|
||||
}
|
||||
// Copy found objects
|
||||
objInfos := make([]ObjectInfo, 0, i+1)
|
||||
for _, objInfo := range objInfoFound {
|
||||
if objInfo == nil {
|
||||
continue
|
||||
}
|
||||
objInfos = append(objInfos, *objInfo)
|
||||
nextMarker = objInfo.Name
|
||||
}
|
||||
|
||||
// Save list routine for the next marker if we haven't reached EOF.
|
||||
params := listParams{bucket, recursive, nextMarker, prefix}
|
||||
if !eof {
|
||||
tpool.Set(params, walkResultCh, endWalkCh)
|
||||
}
|
||||
|
||||
result := ListObjectsInfo{}
|
||||
for _, objInfo := range objInfos {
|
||||
if objInfo.IsDir && delimiter == SlashSeparator {
|
||||
result.Prefixes = append(result.Prefixes, objInfo.Name)
|
||||
continue
|
||||
}
|
||||
result.Objects = append(result.Objects, objInfo)
|
||||
}
|
||||
|
||||
if !eof {
|
||||
result.IsTruncated = true
|
||||
if len(objInfos) > 0 {
|
||||
result.NextMarker = objInfos[len(objInfos)-1].Name
|
||||
} else if len(result.Prefixes) > 0 {
|
||||
result.NextMarker = result.Prefixes[len(result.Prefixes)-1]
|
||||
}
|
||||
}
|
||||
|
||||
// Success.
|
||||
return result, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user