mirrors/object-browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add retention mode and legal hold mode on list objects api (#312)

Browse Source 
Co-authored-by: Daniel Valdivia <hola@danielvaldivia.com>
This commit is contained in:
Cesar N
2020-10-06 16:07:33 -07:00
committed by GitHub
parent dccdfb5533
commit f91346dc5b
8 changed files with 442 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
restapi/client.go
View File

@@ -20,6 +20,7 @@ import (
"context"
"fmt"
"strings"
"time"
"github.com/minio/minio-go/v7/pkg/replication"
@@ -54,6 +55,8 @@ type MinioClient interface {
getBucketNotification(ctx context.Context, bucketName string) (config notification.Configuration, err error)
getBucketPolicy(ctx context.Context, bucketName string) (string, error)
listObjects(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo
getObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error)
getObjectLegalHold(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error)
}
// Interface implementation
@@ -116,6 +119,14 @@ func (c minioClient) listObjects(ctx context.Context, bucket string, opts minio.
return c.client.ListObjects(ctx, bucket, opts)
}
func (c minioClient) getObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error) {
return c.client.GetObjectRetention(ctx, bucketName, objectName, versionID)
}
func (c minioClient) getObjectLegalHold(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error) {
return c.client.GetObjectLegalHold(ctx, bucketName, objectName, opts)
}
// MCClient interface with all functions to be implemented
// by mock when testing, it should include all mc/S3Client respective api calls
// that are used within this project.

70
restapi/embedded_spec.go
View File

@@ -364,6 +364,11 @@ func init() {
"type": "boolean",
"name": "recursive",
"in": "query"
},
{
"type": "boolean",
"name": "with_versions",
"in": "query"
}
],
"responses": {
@@ -2570,15 +2575,45 @@ func init() {
"content_type": {
"type": "string"
},
"expiration": {
"type": "string"
},
"expiration_rule_id": {
"type": "string"
},
"is_delete_marker": {
"type": "boolean"
},
"is_latest": {
"type": "boolean"
},
"last_modified": {
"type": "string"
},
"legal_hold_status": {
"type": "string"
},
"name": {
"type": "string"
},
"retention_mode": {
"type": "string"
},
"retention_until_date": {
"type": "string"
},
"size": {
"type": "integer",
"format": "int64"
},
"user_tags": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"version_id": {
"type": "string"
}
}
},
@@ -4740,6 +4775,11 @@ func init() {
"type": "boolean",
"name": "recursive",
"in": "query"
},
{
"type": "boolean",
"name": "with_versions",
"in": "query"
}
],
"responses": {
@@ -7469,15 +7509,45 @@ func init() {
"content_type": {
"type": "string"
},
"expiration": {
"type": "string"
},
"expiration_rule_id": {
"type": "string"
},
"is_delete_marker": {
"type": "boolean"
},
"is_latest": {
"type": "boolean"
},
"last_modified": {
"type": "string"
},
"legal_hold_status": {
"type": "string"
},
"name": {
"type": "string"
},
"retention_mode": {
"type": "string"
},
"retention_until_date": {
"type": "string"
},
"size": {
"type": "integer",
"format": "int64"
},
"user_tags": {
"type": "object",
"additionalProperties": {
"type": "string"
}
},
"version_id": {
"type": "string"
}
}
},

31
restapi/operations/user_api/list_objects_parameters.go
View File

@@ -61,6 +61,10 @@ type ListObjectsParams struct {
In: query
*/
Recursive *bool
/*
In: query
*/
WithVersions *bool
}
// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface
@@ -89,6 +93,11 @@ func (o *ListObjectsParams) BindRequest(r *http.Request, route *middleware.Match
res = append(res, err)
}
qWithVersions, qhkWithVersions, _ := qs.GetOK("with_versions")
if err := o.bindWithVersions(qWithVersions, qhkWithVersions, route.Formats); err != nil {
res = append(res, err)
}
if len(res) > 0 {
return errors.CompositeValidationError(res...)
}
@@ -149,3 +158,25 @@ func (o *ListObjectsParams) bindRecursive(rawData []string, hasKey bool, formats
return nil
}
// bindWithVersions binds and validates parameter WithVersions from query.
func (o *ListObjectsParams) bindWithVersions(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: false
// AllowEmptyValue: false
if raw == "" { // empty values pass all other validations
return nil
}
value, err := swag.ConvertBool(raw)
if err != nil {
return errors.InvalidType("with_versions", "query", "bool", raw)
}
o.WithVersions = &value
return nil
}

13
restapi/operations/user_api/list_objects_urlbuilder.go
View File

@@ -35,8 +35,9 @@ import (
type ListObjectsURL struct {
BucketName string
Prefix *string
Recursive *bool
Prefix *string
Recursive *bool
WithVersions *bool
_basePath string
// avoid unkeyed usage
@@ -95,6 +96,14 @@ func (o *ListObjectsURL) Build() (*url.URL, error) {
qs.Set("recursive", recursiveQ)
}
var withVersionsQ string
if o.WithVersions != nil {
withVersionsQ = swag.FormatBool(*o.WithVersions)
}
if withVersionsQ != "" {
qs.Set("with_versions", withVersionsQ)
}
_result.RawQuery = qs.Encode()
return &_result, nil

53
restapi/user_objects.go
View File

@@ -19,6 +19,7 @@ package restapi
import (
"context"
"fmt"
"log"
"path/filepath"
"regexp"
"strings"
@@ -29,6 +30,7 @@ import (
"github.com/minio/console/restapi/operations"
"github.com/minio/console/restapi/operations/user_api"
mc "github.com/minio/mc/cmd"
"github.com/minio/mc/pkg/probe"
"github.com/minio/minio-go/v7"
)
@@ -62,12 +64,16 @@ func getListObjectsResponse(session *models.Principal, params user_api.ListObjec
defer cancel()
var prefix string
var recursive bool
var withVersions bool
if params.Prefix != nil {
prefix = *params.Prefix
}
if params.Recursive != nil {
recursive = *params.Recursive
}
if params.WithVersions != nil {
withVersions = *params.WithVersions
}
// bucket request needed to proceed
if params.BucketName == "" {
return nil, prepareError(errBucketNameNotInRequest)
@@ -80,7 +86,7 @@ func getListObjectsResponse(session *models.Principal, params user_api.ListObjec
// defining the client to be used
minioClient := minioClient{client: mClient}
objs, err := listBucketObjects(ctx, minioClient, params.BucketName, prefix, recursive)
objs, err := listBucketObjects(ctx, minioClient, params.BucketName, prefix, recursive, withVersions)
if err != nil {
return nil, prepareError(err)
}
@@ -93,17 +99,50 @@ func getListObjectsResponse(session *models.Principal, params user_api.ListObjec
}
// listBucketObjects gets an array of objects in a bucket
func listBucketObjects(ctx context.Context, client MinioClient, bucketName string, prefix string, recursive bool) ([]*models.BucketObject, error) {
func listBucketObjects(ctx context.Context, client MinioClient, bucketName string, prefix string, recursive, withVersions bool) ([]*models.BucketObject, error) {
var objects []*models.BucketObject
for lsObj := range client.listObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive}) {
for lsObj := range client.listObjects(ctx, bucketName, minio.ListObjectsOptions{Prefix: prefix, Recursive: recursive, WithVersions: withVersions}) {
if lsObj.Err != nil {
return nil, lsObj.Err
}
obj := &models.BucketObject{
Name: lsObj.Key,
Size: lsObj.Size,
LastModified: lsObj.LastModified.String(),
ContentType: lsObj.ContentType,
Name: lsObj.Key,
Size: lsObj.Size,
LastModified: lsObj.LastModified.String(),
ContentType: lsObj.ContentType,
VersionID: lsObj.VersionID,
IsLatest: lsObj.IsLatest,
IsDeleteMarker: lsObj.IsDeleteMarker,
UserTags: lsObj.UserTags,
}
if !lsObj.IsDeleteMarker {
// Add Legal Hold Status if available
legalHoldStatus, err := client.getObjectLegalHold(ctx, bucketName, lsObj.Key, minio.GetObjectLegalHoldOptions{VersionID: lsObj.VersionID})
if err != nil {
errResp := minio.ToErrorResponse(probe.NewError(err).ToGoError())
if errResp.Code != "NoSuchObjectLockConfiguration" {
log.Printf("error getting legal hold status for %s : %s", lsObj.VersionID, err)
}
} else {
if legalHoldStatus != nil {
obj.LegalHoldStatus = string(*legalHoldStatus)
}
}
// Add Retention Status if available
retention, retUntilDate, err := client.getObjectRetention(ctx, bucketName, lsObj.Key, lsObj.VersionID)
if err != nil {
errResp := minio.ToErrorResponse(probe.NewError(err).ToGoError())
if errResp.Code != "NoSuchObjectLockConfiguration" {
log.Printf("error getting retention status for %s : %s", lsObj.VersionID, err)
}
} else {
if retention != nil && retUntilDate != nil {
date := *retUntilDate
obj.RetentionMode = string(*retention)
obj.RetentionUntilDate = date.String()
}
}
}
objects = append(objects, obj)
}

248
restapi/user_objects_test.go
View File

@@ -31,20 +31,28 @@ import (
)
var minioListObjectsMock func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo
var minioGetObjectLegalHoldMock func(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error)
var minioGetObjectRetentionMock func(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error)
var mcListMock func(ctx context.Context, opts mc.ListOptions) <-chan *mc.ClientContent
var mcRemoveMock func(ctx context.Context, isIncomplete, isRemoveBucket, isBypass bool, contentCh <-chan *mc.ClientContent) <-chan *probe.Error
// mock function of listObjects() needed for list objects
// mock functions for minioClientMock
func (ac minioClientMock) listObjects(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo {
return minioListObjectsMock(ctx, bucket, opts)
}
func (ac minioClientMock) getObjectLegalHold(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error) {
return minioGetObjectLegalHoldMock(ctx, bucketName, objectName, opts)
}
// implements mc.S3Client.List()
func (ac minioClientMock) getObjectRetention(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error) {
return minioGetObjectRetentionMock(ctx, bucketName, objectName, versionID)
}
// mock functions for s3ClientMock
func (c s3ClientMock) list(ctx context.Context, opts mc.ListOptions) <-chan *mc.ClientContent {
return mcListMock(ctx, opts)
}
// implements mc.S3Client.Remove()
func (c s3ClientMock) remove(ctx context.Context, isIncomplete, isRemoveBucket, isBypass bool, contentCh <-chan *mc.ClientContent) <-chan *probe.Error {
return mcRemoveMock(ctx, isIncomplete, isRemoveBucket, isBypass, contentCh)
}
@@ -52,12 +60,16 @@ func (c s3ClientMock) remove(ctx context.Context, isIncomplete, isRemoveBucket,
func Test_listObjects(t *testing.T) {
ctx := context.Background()
t1 := time.Now()
tretention := time.Now()
minClient := minioClientMock{}
type args struct {
bucketName string
prefix string
recursive bool
listFunc func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo
bucketName string
prefix string
recursive bool
withVersions bool
listFunc func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo
objectLegalHoldFunc func(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error)
objectRetentionFunc func(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error)
}
tests := []struct {
test string
@@ -68,9 +80,10 @@ func Test_listObjects(t *testing.T) {
{
test: "Return objects",
args: args{
bucketName: "bucket1",
prefix: "prefix",
recursive: true,
bucketName: "bucket1",
prefix: "prefix",
recursive: true,
withVersions: false,
listFunc: func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo {
objectStatCh := make(chan minio.ObjectInfo, 1)
go func(objectStatCh chan<- minio.ObjectInfo) {
@@ -94,18 +107,32 @@ func Test_listObjects(t *testing.T) {
}(objectStatCh)
return objectStatCh
},
objectLegalHoldFunc: func(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error) {
s := minio.LegalHoldEnabled
return &s, nil
},
objectRetentionFunc: func(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error) {
m := minio.Governance
return &m, &tretention, nil
},
},
expectedResp: []*models.BucketObject{
&models.BucketObject{
Name: "obj1",
LastModified: t1.String(),
Size: int64(1024),
ContentType: "content",
Name: "obj1",
LastModified: t1.String(),
Size: int64(1024),
ContentType: "content",
LegalHoldStatus: string(minio.LegalHoldEnabled),
RetentionMode: string(minio.Governance),
RetentionUntilDate: tretention.String(),
}, &models.BucketObject{
Name: "obj2",
LastModified: t1.String(),
Size: int64(512),
ContentType: "content",
Name: "obj2",
LastModified: t1.String(),
Size: int64(512),
ContentType: "content",
LegalHoldStatus: string(minio.LegalHoldEnabled),
RetentionMode: string(minio.Governance),
RetentionUntilDate: tretention.String(),
},
},
wantError: nil,
@@ -113,14 +140,23 @@ func Test_listObjects(t *testing.T) {
{
test: "Return zero objects",
args: args{
bucketName: "bucket1",
prefix: "prefix",
recursive: true,
bucketName: "bucket1",
prefix: "prefix",
recursive: true,
withVersions: false,
listFunc: func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo {
objectStatCh := make(chan minio.ObjectInfo, 1)
defer close(objectStatCh)
return objectStatCh
},
objectLegalHoldFunc: func(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error) {
s := minio.LegalHoldEnabled
return &s, nil
},
objectRetentionFunc: func(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error) {
m := minio.Governance
return &m, &tretention, nil
},
},
expectedResp: nil,
wantError: nil,
@@ -128,9 +164,10 @@ func Test_listObjects(t *testing.T) {
{
test: "Handle error if present on object",
args: args{
bucketName: "bucket1",
prefix: "prefix",
recursive: true,
bucketName: "bucket1",
prefix: "prefix",
recursive: true,
withVersions: false,
listFunc: func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo {
objectStatCh := make(chan minio.ObjectInfo, 1)
go func(objectStatCh chan<- minio.ObjectInfo) {
@@ -151,16 +188,175 @@ func Test_listObjects(t *testing.T) {
}(objectStatCh)
return objectStatCh
},
objectLegalHoldFunc: func(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error) {
s := minio.LegalHoldEnabled
return &s, nil
},
objectRetentionFunc: func(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error) {
m := minio.Governance
return &m, &tretention, nil
},
},
expectedResp: nil,
wantError: errors.New("error here"),
},
{
// Description: deleted objects with IsDeleteMarker
// should not call legsalhold or retention funcs
test: "Return deleted objects",
args: args{
bucketName: "bucket1",
prefix: "prefix",
recursive: true,
withVersions: false,
listFunc: func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo {
objectStatCh := make(chan minio.ObjectInfo, 1)
go func(objectStatCh chan<- minio.ObjectInfo) {
defer close(objectStatCh)
for _, bucket := range []minio.ObjectInfo{
minio.ObjectInfo{
Key: "obj1",
LastModified: t1,
Size: int64(1024),
ContentType: "content",
IsDeleteMarker: true,
},
minio.ObjectInfo{
Key: "obj2",
LastModified: t1,
Size: int64(512),
ContentType: "content",
},
} {
objectStatCh <- bucket
}
}(objectStatCh)
return objectStatCh
},
objectLegalHoldFunc: func(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error) {
s := minio.LegalHoldEnabled
return &s, nil
},
objectRetentionFunc: func(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error) {
m := minio.Governance
return &m, &tretention, nil
},
},
expectedResp: []*models.BucketObject{
&models.BucketObject{
Name: "obj1",
LastModified: t1.String(),
Size: int64(1024),
ContentType: "content",
IsDeleteMarker: true,
}, &models.BucketObject{
Name: "obj2",
LastModified: t1.String(),
Size: int64(512),
ContentType: "content",
LegalHoldStatus: string(minio.LegalHoldEnabled),
RetentionMode: string(minio.Governance),
RetentionUntilDate: tretention.String(),
},
},
wantError: nil,
},
{
// Description: deleted objects with
// error on legalhold and retention funcs
// should only log errors
test: "Return deleted objects, error on legalhold and retention",
args: args{
bucketName: "bucket1",
prefix: "prefix",
recursive: true,
withVersions: false,
listFunc: func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo {
objectStatCh := make(chan minio.ObjectInfo, 1)
go func(objectStatCh chan<- minio.ObjectInfo) {
defer close(objectStatCh)
for _, bucket := range []minio.ObjectInfo{
minio.ObjectInfo{
Key: "obj1",
LastModified: t1,
Size: int64(1024),
ContentType: "content",
},
} {
objectStatCh <- bucket
}
}(objectStatCh)
return objectStatCh
},
objectLegalHoldFunc: func(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error) {
return nil, errors.New("error legal")
},
objectRetentionFunc: func(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error) {
return nil, nil, errors.New("error retention")
},
},
expectedResp: []*models.BucketObject{
&models.BucketObject{
Name: "obj1",
LastModified: t1.String(),
Size: int64(1024),
ContentType: "content",
},
},
wantError: nil,
},
{
// Description: deleted objects with
// error on legalhold and retention funcs
// should only log errors
test: "Return deleted objects, error on legalhold and retention",
args: args{
bucketName: "bucket1",
prefix: "prefix",
recursive: true,
withVersions: false,
listFunc: func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo {
objectStatCh := make(chan minio.ObjectInfo, 1)
go func(objectStatCh chan<- minio.ObjectInfo) {
defer close(objectStatCh)
for _, bucket := range []minio.ObjectInfo{
minio.ObjectInfo{
Key: "obj1",
LastModified: t1,
Size: int64(1024),
ContentType: "content",
},
} {
objectStatCh <- bucket
}
}(objectStatCh)
return objectStatCh
},
objectLegalHoldFunc: func(ctx context.Context, bucketName, objectName string, opts minio.GetObjectLegalHoldOptions) (status *minio.LegalHoldStatus, err error) {
return nil, errors.New("error legal")
},
objectRetentionFunc: func(ctx context.Context, bucketName, objectName, versionID string) (mode *minio.RetentionMode, retainUntilDate *time.Time, err error) {
return nil, nil, errors.New("error retention")
},
},
expectedResp: []*models.BucketObject{
&models.BucketObject{
Name: "obj1",
LastModified: t1.String(),
Size: int64(1024),
ContentType: "content",
},
},
wantError: nil,
},
}
for _, tt := range tests {
t.Run(tt.test, func(t *testing.T) {
minioListObjectsMock = tt.args.listFunc
resp, err := listBucketObjects(ctx, minClient, tt.args.bucketName, tt.args.prefix, tt.args.recursive)
minioGetObjectLegalHoldMock = tt.args.objectLegalHoldFunc
minioGetObjectRetentionMock = tt.args.objectRetentionFunc
resp, err := listBucketObjects(ctx, minClient, tt.args.bucketName, tt.args.prefix, tt.args.recursive, tt.args.withVersions)
if !reflect.DeepEqual(err, tt.wantError) {
t.Errorf("listBucketObjects() error: %v, wantErr: %v", err, tt.wantError)
return