From b968cc25add0854847cd377a4f78aa58ae54427f Mon Sep 17 00:00:00 2001 From: Prakash Senthil Vel <23444145+prakashsvmx@users.noreply.github.com> Date: Thu, 3 Aug 2023 05:58:25 +0530 Subject: [PATCH] feat: download multiple object selection as zip ignoring any deleted objects selected (#2965) --- integration/objects_test.go | 85 +++++++++++ portal-ui/src/api/consoleApi.ts | 22 +++ .../Objects/ListObjects/ListObjects.tsx | 7 +- .../Buckets/ListBuckets/Objects/utils.ts | 55 +++++-- .../ObjectBrowser/objectBrowserThunks.ts | 39 ++++- restapi/embedded_spec.go | 106 ++++++++++++++ restapi/operations/console_api.go | 12 ++ .../object/download_multiple_objects.go | 88 ++++++++++++ .../download_multiple_objects_parameters.go | 112 +++++++++++++++ .../download_multiple_objects_responses.go | 134 ++++++++++++++++++ .../download_multiple_objects_urlbuilder.go | 116 +++++++++++++++ restapi/user_objects.go | 134 ++++++++++++++++++ restapi/user_objects_test.go | 87 ++++++++++-- swagger.yml | 33 +++++ 14 files changed, 1002 insertions(+), 28 deletions(-) create mode 100644 restapi/operations/object/download_multiple_objects.go create mode 100644 restapi/operations/object/download_multiple_objects_parameters.go create mode 100644 restapi/operations/object/download_multiple_objects_responses.go create mode 100644 restapi/operations/object/download_multiple_objects_urlbuilder.go diff --git a/integration/objects_test.go b/integration/objects_test.go index b80ff4976..e1882797f 100644 --- a/integration/objects_test.go +++ b/integration/objects_test.go @@ -17,8 +17,10 @@ package integration import ( + "bytes" "context" "encoding/base64" + "encoding/json" "fmt" "log" "math/rand" @@ -194,3 +196,86 @@ func TestObjectGet(t *testing.T) { }) } } + +func downloadMultipleFiles(bucketName string, objects []string) (*http.Response, error) { + requestURL := fmt.Sprintf("http://localhost:9090/api/v1/buckets/%s/objects/download-multiple", bucketName) + + postReqParams, _ := json.Marshal(objects) + reqBody := bytes.NewReader(postReqParams) + + request, err := http.NewRequest( + "POST", requestURL, reqBody) + if err != nil { + log.Println(err) + return nil, nil + } + + request.Header.Add("Cookie", fmt.Sprintf("token=%s", token)) + request.Header.Add("Content-Type", "application/json") + client := &http.Client{ + Timeout: 2 * time.Second, + } + response, err := client.Do(request) + return response, err +} + +func TestDownloadMultipleFiles(t *testing.T) { + assert := assert.New(t) + type args struct { + bucketName string + objectLis []string + } + tests := []struct { + name string + args args + expectedStatus int + expectedError bool + }{ + { + name: "Test empty Bucket", + args: args{ + bucketName: "", + }, + expectedStatus: 400, + expectedError: true, + }, + { + name: "Test empty object list", + args: args{ + bucketName: "test-bucket", + }, + expectedStatus: 400, + expectedError: true, + }, + { + name: "Test with bucket and object list", + args: args{ + bucketName: "test-bucket", + objectLis: []string{ + "my-object.txt", + "test-prefix/", + "test-prefix/nested-prefix/", + "test-prefix/nested-prefix/deep-nested/", + }, + }, + expectedStatus: 200, + expectedError: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resp, err := downloadMultipleFiles(tt.args.bucketName, tt.args.objectLis) + if tt.expectedError { + assert.Nil(err) + if err != nil { + log.Println(err) + return + } + } + if resp != nil { + assert.NotNil(resp) + } + }) + } +} diff --git a/portal-ui/src/api/consoleApi.ts b/portal-ui/src/api/consoleApi.ts index ec3c01f58..1b65864d9 100644 --- a/portal-ui/src/api/consoleApi.ts +++ b/portal-ui/src/api/consoleApi.ts @@ -2142,6 +2142,28 @@ export class Api< ...params, }), + /** + * No description + * + * @tags Object + * @name DownloadMultipleObjects + * @summary Download Multiple Objects + * @request POST:/buckets/{bucket_name}/objects/download-multiple + * @secure + */ + downloadMultipleObjects: ( + bucketName: string, + objectList: string[], + params: RequestParams = {}, + ) => + this.request({ + path: `/buckets/${bucketName}/objects/download-multiple`, + method: "POST", + body: objectList, + secure: true, + ...params, + }), + /** * No description * diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx index 381bec086..59a86db35 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx @@ -912,6 +912,11 @@ const ListObjects = () => { createdTime = DateTime.fromISO(bucketInfo.creation_date); } + const downloadToolTip = + selectedObjects?.length <= 1 + ? "Download Selected" + : ` Download selected objects as Zip. Any Deleted objects in the selection would be skipped from download.`; + const multiActionButtons = [ { action: () => { @@ -921,7 +926,7 @@ const ListObjects = () => { disabled: !canDownload || selectedObjects?.length === 0, icon: , tooltip: canDownload - ? "Download Selected" + ? downloadToolTip : permissionTooltipHelper( [IAM_SCOPES.S3_GET_OBJECT, IAM_SCOPES.S3_GET_ACTIONS], "download objects from this bucket", diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts index b50693355..58d69a7fd 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts @@ -18,8 +18,52 @@ import { BucketObjectItem } from "./ListObjects/types"; import { encodeURLString } from "../../../../../common/utils"; import { removeTrace } from "../../../ObjectBrowser/transferManager"; import store from "../../../../../store"; -import { PermissionResource } from "api/consoleApi"; +import { ContentType, PermissionResource } from "api/consoleApi"; +import { api } from "../../../../../api"; +import { setErrorSnackMessage } from "../../../../../systemSlice"; +const downloadWithLink = (href: string, downloadFileName: string) => { + const link = document.createElement("a"); + link.href = href; + link.download = downloadFileName; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); +}; + +export const downloadSelectedAsZip = async ( + bucketName: string, + objectList: string[], + resultFileName: string, +) => { + const state = store.getState(); + const anonymousMode = state.system.anonymousMode; + + try { + const resp = await api.buckets.downloadMultipleObjects( + bucketName, + objectList, + { + type: ContentType.Json, + headers: anonymousMode + ? { + "X-Anonymous": "1", + } + : undefined, + }, + ); + const blob = await resp.blob(); + const href = window.URL.createObjectURL(blob); + downloadWithLink(href, resultFileName); + } catch (err: any) { + store.dispatch( + setErrorSnackMessage({ + errorMessage: `Download of multiple files failed. ${err.statusText}`, + detailedError: "", + }), + ); + } +}; export const download = ( bucketName: string, objectPath: string, @@ -33,8 +77,6 @@ export const download = ( abortCallback: () => void, toastCallback: () => void, ) => { - const anchor = document.createElement("a"); - document.body.appendChild(anchor); let basename = document.baseURI.replace(window.location.origin, ""); const state = store.getState(); const anonymousMode = state.system.anonymousMode; @@ -90,12 +132,7 @@ export const download = ( removeTrace(id); - var link = document.createElement("a"); - link.href = window.URL.createObjectURL(req.response); - link.download = filename; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); + downloadWithLink(window.URL.createObjectURL(req.response), filename); } else { if (req.getResponseHeader("Content-Type") === "application/json") { const rspBody: { detailedMessage?: string } = JSON.parse( diff --git a/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts b/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts index b31d593c3..9ddff6d95 100644 --- a/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts +++ b/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts @@ -19,7 +19,10 @@ import { AppState } from "../../../store"; import { encodeURLString, getClientOS } from "../../../common/utils"; import { BucketObjectItem } from "../Buckets/ListBuckets/Objects/ListObjects/types"; import { makeid, storeCallForObjectWithID } from "./transferManager"; -import { download } from "../Buckets/ListBuckets/Objects/utils"; +import { + download, + downloadSelectedAsZip, +} from "../Buckets/ListBuckets/Objects/utils"; import { cancelObjectInList, completeObject, @@ -33,6 +36,7 @@ import { updateProgress, } from "./objectBrowserSlice"; import { setSnackBarMessage } from "../../../systemSlice"; +import { DateTime } from "luxon"; export const downloadSelected = createAsyncThunk( "objectBrowser/downloadSelected", @@ -104,8 +108,7 @@ export const downloadSelected = createAsyncThunk( itemsToDownload = state.objectBrowser.records.filter(filterFunction); - // I case just one element is selected, then we trigger download modal validation. - // We are going to enforce zip download when multiple files are selected + // In case just one element is selected, then we trigger download modal validation. if (itemsToDownload.length === 1) { if ( itemsToDownload[0].name.length > 200 && @@ -113,12 +116,34 @@ export const downloadSelected = createAsyncThunk( ) { dispatch(setDownloadRenameModal(itemsToDownload[0])); return; + } else { + downloadObject(itemsToDownload[0]); + } + } else { + if (itemsToDownload.length === 1) { + downloadObject(itemsToDownload[0]); + } else if (itemsToDownload.length > 1) { + const fileName = `${DateTime.now().toFormat( + "LL-dd-yyyy-HH-mm-ss", + )}_files_list.zip`; + + // We are enforcing zip download when multiple files are selected for better user experience + const multiObjList = itemsToDownload.reduce((dwList: any[], bi) => { + // Download objects/prefixes(recursively) as zip + // Skip any deleted files selected via "Show deleted objects" in selection and log for debugging + const isDeleted = bi?.delete_flag; + if (bi && !isDeleted) { + dwList.push(bi.name); + } else { + console.log(`Skipping ${bi?.name} from download.`); + } + return dwList; + }, []); + + await downloadSelectedAsZip(bucketName, multiObjList, fileName); + return; } } - - itemsToDownload.forEach((filteredItem) => { - downloadObject(filteredItem); - }); } }, ); diff --git a/restapi/embedded_spec.go b/restapi/embedded_spec.go index 3238fc1f4..fbe94618c 100644 --- a/restapi/embedded_spec.go +++ b/restapi/embedded_spec.go @@ -1646,6 +1646,59 @@ func init() { } } }, + "/buckets/{bucket_name}/objects/download-multiple": { + "post": { + "security": [ + { + "key": [] + }, + { + "anonymous": [] + } + ], + "produces": [ + "application/octet-stream" + ], + "tags": [ + "Object" + ], + "summary": "Download Multiple Objects", + "operationId": "DownloadMultipleObjects", + "parameters": [ + { + "type": "string", + "name": "bucket_name", + "in": "path", + "required": true + }, + { + "name": "objectList", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + ], + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "type": "file" + } + }, + "default": { + "description": "Generic error response.", + "schema": { + "$ref": "#/definitions/error" + } + } + } + } + }, "/buckets/{bucket_name}/objects/legalhold": { "put": { "tags": [ @@ -10660,6 +10713,59 @@ func init() { } } }, + "/buckets/{bucket_name}/objects/download-multiple": { + "post": { + "security": [ + { + "key": [] + }, + { + "anonymous": [] + } + ], + "produces": [ + "application/octet-stream" + ], + "tags": [ + "Object" + ], + "summary": "Download Multiple Objects", + "operationId": "DownloadMultipleObjects", + "parameters": [ + { + "type": "string", + "name": "bucket_name", + "in": "path", + "required": true + }, + { + "name": "objectList", + "in": "body", + "required": true, + "schema": { + "type": "array", + "items": { + "type": "string" + } + } + } + ], + "responses": { + "200": { + "description": "A successful response.", + "schema": { + "type": "file" + } + }, + "default": { + "description": "Generic error response.", + "schema": { + "$ref": "#/definitions/error" + } + } + } + } + }, "/buckets/{bucket_name}/objects/legalhold": { "put": { "tags": [ diff --git a/restapi/operations/console_api.go b/restapi/operations/console_api.go index e3cf2d745..14072c61c 100644 --- a/restapi/operations/console_api.go +++ b/restapi/operations/console_api.go @@ -211,6 +211,9 @@ func NewConsoleAPI(spec *loads.Document) *ConsoleAPI { ObjectDownloadObjectHandler: object.DownloadObjectHandlerFunc(func(params object.DownloadObjectParams, principal *models.Principal) middleware.Responder { return middleware.NotImplemented("operation object.DownloadObject has not yet been implemented") }), + ObjectDownloadMultipleObjectsHandler: object.DownloadMultipleObjectsHandlerFunc(func(params object.DownloadMultipleObjectsParams, principal *models.Principal) middleware.Responder { + return middleware.NotImplemented("operation object.DownloadMultipleObjects has not yet been implemented") + }), TieringEditTierCredentialsHandler: tiering.EditTierCredentialsHandlerFunc(func(params tiering.EditTierCredentialsParams, principal *models.Principal) middleware.Responder { return middleware.NotImplemented("operation tiering.EditTierCredentials has not yet been implemented") }), @@ -701,6 +704,8 @@ type ConsoleAPI struct { BucketDisableBucketEncryptionHandler bucket.DisableBucketEncryptionHandler // ObjectDownloadObjectHandler sets the operation handler for the download object operation ObjectDownloadObjectHandler object.DownloadObjectHandler + // ObjectDownloadMultipleObjectsHandler sets the operation handler for the download multiple objects operation + ObjectDownloadMultipleObjectsHandler object.DownloadMultipleObjectsHandler // TieringEditTierCredentialsHandler sets the operation handler for the edit tier credentials operation TieringEditTierCredentialsHandler tiering.EditTierCredentialsHandler // BucketEnableBucketEncryptionHandler sets the operation handler for the enable bucket encryption operation @@ -1145,6 +1150,9 @@ func (o *ConsoleAPI) Validate() error { if o.ObjectDownloadObjectHandler == nil { unregistered = append(unregistered, "object.DownloadObjectHandler") } + if o.ObjectDownloadMultipleObjectsHandler == nil { + unregistered = append(unregistered, "object.DownloadMultipleObjectsHandler") + } if o.TieringEditTierCredentialsHandler == nil { unregistered = append(unregistered, "tiering.EditTierCredentialsHandler") } @@ -1761,6 +1769,10 @@ func (o *ConsoleAPI) initHandlerCache() { o.handlers["GET"] = make(map[string]http.Handler) } o.handlers["GET"]["/buckets/{bucket_name}/objects/download"] = object.NewDownloadObject(o.context, o.ObjectDownloadObjectHandler) + if o.handlers["POST"] == nil { + o.handlers["POST"] = make(map[string]http.Handler) + } + o.handlers["POST"]["/buckets/{bucket_name}/objects/download-multiple"] = object.NewDownloadMultipleObjects(o.context, o.ObjectDownloadMultipleObjectsHandler) if o.handlers["PUT"] == nil { o.handlers["PUT"] = make(map[string]http.Handler) } diff --git a/restapi/operations/object/download_multiple_objects.go b/restapi/operations/object/download_multiple_objects.go new file mode 100644 index 000000000..2f54b7b49 --- /dev/null +++ b/restapi/operations/object/download_multiple_objects.go @@ -0,0 +1,88 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// This file is part of MinIO Console Server +// Copyright (c) 2023 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// + +package object + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "net/http" + + "github.com/go-openapi/runtime/middleware" + + "github.com/minio/console/models" +) + +// DownloadMultipleObjectsHandlerFunc turns a function with the right signature into a download multiple objects handler +type DownloadMultipleObjectsHandlerFunc func(DownloadMultipleObjectsParams, *models.Principal) middleware.Responder + +// Handle executing the request and returning a response +func (fn DownloadMultipleObjectsHandlerFunc) Handle(params DownloadMultipleObjectsParams, principal *models.Principal) middleware.Responder { + return fn(params, principal) +} + +// DownloadMultipleObjectsHandler interface for that can handle valid download multiple objects params +type DownloadMultipleObjectsHandler interface { + Handle(DownloadMultipleObjectsParams, *models.Principal) middleware.Responder +} + +// NewDownloadMultipleObjects creates a new http.Handler for the download multiple objects operation +func NewDownloadMultipleObjects(ctx *middleware.Context, handler DownloadMultipleObjectsHandler) *DownloadMultipleObjects { + return &DownloadMultipleObjects{Context: ctx, Handler: handler} +} + +/* + DownloadMultipleObjects swagger:route POST /buckets/{bucket_name}/objects/download-multiple Object downloadMultipleObjects + +Download Multiple Objects +*/ +type DownloadMultipleObjects struct { + Context *middleware.Context + Handler DownloadMultipleObjectsHandler +} + +func (o *DownloadMultipleObjects) ServeHTTP(rw http.ResponseWriter, r *http.Request) { + route, rCtx, _ := o.Context.RouteInfo(r) + if rCtx != nil { + *r = *rCtx + } + var Params = NewDownloadMultipleObjectsParams() + uprinc, aCtx, err := o.Context.Authorize(r, route) + if err != nil { + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + if aCtx != nil { + *r = *aCtx + } + var principal *models.Principal + if uprinc != nil { + principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise + } + + if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params + o.Context.Respond(rw, r, route.Produces, route, err) + return + } + + res := o.Handler.Handle(Params, principal) // actually handle the request + o.Context.Respond(rw, r, route.Produces, route, res) + +} diff --git a/restapi/operations/object/download_multiple_objects_parameters.go b/restapi/operations/object/download_multiple_objects_parameters.go new file mode 100644 index 000000000..8831d2f10 --- /dev/null +++ b/restapi/operations/object/download_multiple_objects_parameters.go @@ -0,0 +1,112 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// This file is part of MinIO Console Server +// Copyright (c) 2023 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// + +package object + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "io" + "net/http" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + "github.com/go-openapi/runtime/middleware" + "github.com/go-openapi/strfmt" +) + +// NewDownloadMultipleObjectsParams creates a new DownloadMultipleObjectsParams object +// +// There are no default values defined in the spec. +func NewDownloadMultipleObjectsParams() DownloadMultipleObjectsParams { + + return DownloadMultipleObjectsParams{} +} + +// DownloadMultipleObjectsParams contains all the bound params for the download multiple objects operation +// typically these are obtained from a http.Request +// +// swagger:parameters DownloadMultipleObjects +type DownloadMultipleObjectsParams struct { + + // HTTP Request Object + HTTPRequest *http.Request `json:"-"` + + /* + Required: true + In: path + */ + BucketName string + /* + Required: true + In: body + */ + ObjectList []string +} + +// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface +// for simple values it will use straight method calls. +// +// To ensure default values, the struct must have been initialized with NewDownloadMultipleObjectsParams() beforehand. +func (o *DownloadMultipleObjectsParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error { + var res []error + + o.HTTPRequest = r + + rBucketName, rhkBucketName, _ := route.Params.GetOK("bucket_name") + if err := o.bindBucketName(rBucketName, rhkBucketName, route.Formats); err != nil { + res = append(res, err) + } + + if runtime.HasBody(r) { + defer r.Body.Close() + var body []string + if err := route.Consumer.Consume(r.Body, &body); err != nil { + if err == io.EOF { + res = append(res, errors.Required("objectList", "body", "")) + } else { + res = append(res, errors.NewParseError("objectList", "body", "", err)) + } + } else { + // no validation required on inline body + o.ObjectList = body + } + } else { + res = append(res, errors.Required("objectList", "body", "")) + } + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} + +// bindBucketName binds and validates parameter BucketName from path. +func (o *DownloadMultipleObjectsParams) bindBucketName(rawData []string, hasKey bool, formats strfmt.Registry) error { + var raw string + if len(rawData) > 0 { + raw = rawData[len(rawData)-1] + } + + // Required: true + // Parameter is provided by construction from the route + o.BucketName = raw + + return nil +} diff --git a/restapi/operations/object/download_multiple_objects_responses.go b/restapi/operations/object/download_multiple_objects_responses.go new file mode 100644 index 000000000..eeeff812b --- /dev/null +++ b/restapi/operations/object/download_multiple_objects_responses.go @@ -0,0 +1,134 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// This file is part of MinIO Console Server +// Copyright (c) 2023 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// + +package object + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "io" + "net/http" + + "github.com/go-openapi/runtime" + + "github.com/minio/console/models" +) + +// DownloadMultipleObjectsOKCode is the HTTP code returned for type DownloadMultipleObjectsOK +const DownloadMultipleObjectsOKCode int = 200 + +/* +DownloadMultipleObjectsOK A successful response. + +swagger:response downloadMultipleObjectsOK +*/ +type DownloadMultipleObjectsOK struct { + + /* + In: Body + */ + Payload io.ReadCloser `json:"body,omitempty"` +} + +// NewDownloadMultipleObjectsOK creates DownloadMultipleObjectsOK with default headers values +func NewDownloadMultipleObjectsOK() *DownloadMultipleObjectsOK { + + return &DownloadMultipleObjectsOK{} +} + +// WithPayload adds the payload to the download multiple objects o k response +func (o *DownloadMultipleObjectsOK) WithPayload(payload io.ReadCloser) *DownloadMultipleObjectsOK { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the download multiple objects o k response +func (o *DownloadMultipleObjectsOK) SetPayload(payload io.ReadCloser) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *DownloadMultipleObjectsOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(200) + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } +} + +/* +DownloadMultipleObjectsDefault Generic error response. + +swagger:response downloadMultipleObjectsDefault +*/ +type DownloadMultipleObjectsDefault struct { + _statusCode int + + /* + In: Body + */ + Payload *models.Error `json:"body,omitempty"` +} + +// NewDownloadMultipleObjectsDefault creates DownloadMultipleObjectsDefault with default headers values +func NewDownloadMultipleObjectsDefault(code int) *DownloadMultipleObjectsDefault { + if code <= 0 { + code = 500 + } + + return &DownloadMultipleObjectsDefault{ + _statusCode: code, + } +} + +// WithStatusCode adds the status to the download multiple objects default response +func (o *DownloadMultipleObjectsDefault) WithStatusCode(code int) *DownloadMultipleObjectsDefault { + o._statusCode = code + return o +} + +// SetStatusCode sets the status to the download multiple objects default response +func (o *DownloadMultipleObjectsDefault) SetStatusCode(code int) { + o._statusCode = code +} + +// WithPayload adds the payload to the download multiple objects default response +func (o *DownloadMultipleObjectsDefault) WithPayload(payload *models.Error) *DownloadMultipleObjectsDefault { + o.Payload = payload + return o +} + +// SetPayload sets the payload to the download multiple objects default response +func (o *DownloadMultipleObjectsDefault) SetPayload(payload *models.Error) { + o.Payload = payload +} + +// WriteResponse to the client +func (o *DownloadMultipleObjectsDefault) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) { + + rw.WriteHeader(o._statusCode) + if o.Payload != nil { + payload := o.Payload + if err := producer.Produce(rw, payload); err != nil { + panic(err) // let the recovery middleware deal with this + } + } +} diff --git a/restapi/operations/object/download_multiple_objects_urlbuilder.go b/restapi/operations/object/download_multiple_objects_urlbuilder.go new file mode 100644 index 000000000..051e61f8c --- /dev/null +++ b/restapi/operations/object/download_multiple_objects_urlbuilder.go @@ -0,0 +1,116 @@ +// Code generated by go-swagger; DO NOT EDIT. + +// This file is part of MinIO Console Server +// Copyright (c) 2023 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +// + +package object + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the generate command + +import ( + "errors" + "net/url" + golangswaggerpaths "path" + "strings" +) + +// DownloadMultipleObjectsURL generates an URL for the download multiple objects operation +type DownloadMultipleObjectsURL struct { + BucketName string + + _basePath string + // avoid unkeyed usage + _ struct{} +} + +// WithBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *DownloadMultipleObjectsURL) WithBasePath(bp string) *DownloadMultipleObjectsURL { + o.SetBasePath(bp) + return o +} + +// SetBasePath sets the base path for this url builder, only required when it's different from the +// base path specified in the swagger spec. +// When the value of the base path is an empty string +func (o *DownloadMultipleObjectsURL) SetBasePath(bp string) { + o._basePath = bp +} + +// Build a url path and query string +func (o *DownloadMultipleObjectsURL) Build() (*url.URL, error) { + var _result url.URL + + var _path = "/buckets/{bucket_name}/objects/download-multiple" + + bucketName := o.BucketName + if bucketName != "" { + _path = strings.Replace(_path, "{bucket_name}", bucketName, -1) + } else { + return nil, errors.New("bucketName is required on DownloadMultipleObjectsURL") + } + + _basePath := o._basePath + if _basePath == "" { + _basePath = "/api/v1" + } + _result.Path = golangswaggerpaths.Join(_basePath, _path) + + return &_result, nil +} + +// Must is a helper function to panic when the url builder returns an error +func (o *DownloadMultipleObjectsURL) Must(u *url.URL, err error) *url.URL { + if err != nil { + panic(err) + } + if u == nil { + panic("url can't be nil") + } + return u +} + +// String returns the string representation of the path with query string +func (o *DownloadMultipleObjectsURL) String() string { + return o.Must(o.Build()).String() +} + +// BuildFull builds a full url with scheme, host, path and query string +func (o *DownloadMultipleObjectsURL) BuildFull(scheme, host string) (*url.URL, error) { + if scheme == "" { + return nil, errors.New("scheme is required for a full url on DownloadMultipleObjectsURL") + } + if host == "" { + return nil, errors.New("host is required for a full url on DownloadMultipleObjectsURL") + } + + base, err := o.Build() + if err != nil { + return nil, err + } + + base.Scheme = scheme + base.Host = host + return base, nil +} + +// StringFull returns the string representation of a complete url +func (o *DownloadMultipleObjectsURL) StringFull(scheme, host string) string { + return o.Must(o.BuildFull(scheme, host)).String() +} diff --git a/restapi/user_objects.go b/restapi/user_objects.go index 54e768b5b..43a106294 100644 --- a/restapi/user_objects.go +++ b/restapi/user_objects.go @@ -107,6 +107,21 @@ func registerObjectsHandlers(api *operations.ConsoleAPI) { } return resp }) + // download multiple objects + api.ObjectDownloadMultipleObjectsHandler = objectApi.DownloadMultipleObjectsHandlerFunc(func(params objectApi.DownloadMultipleObjectsParams, session *models.Principal) middleware.Responder { + ctx := params.HTTPRequest.Context() + if len(params.ObjectList) < 1 { + return objectApi.NewDownloadMultipleObjectsDefault(400).WithPayload(ErrorWithContext(ctx, errors.New("could not download, since object list is empty"))) + } + var resp middleware.Responder + var err *models.Error + resp, err = getMultipleFilesDownloadResponse(session, params) + if err != nil { + return objectApi.NewDownloadMultipleObjectsDefault(int(err.Code)).WithPayload(err) + } + return resp + }) + // upload object api.ObjectPostBucketsBucketNameObjectsUploadHandler = objectApi.PostBucketsBucketNameObjectsUploadHandlerFunc(func(params objectApi.PostBucketsBucketNameObjectsUploadParams, session *models.Principal) middleware.Responder { if err := getUploadObjectResponse(session, params); err != nil { @@ -620,6 +635,125 @@ func getDownloadFolderResponse(session *models.Principal, params objectApi.Downl }), nil } +func getMultipleFilesDownloadResponse(session *models.Principal, params objectApi.DownloadMultipleObjectsParams) (middleware.Responder, *models.Error) { + ctx := params.HTTPRequest.Context() + mClient, err := newMinioClient(session, getClientIP(params.HTTPRequest)) + if err != nil { + return nil, ErrorWithContext(ctx, err) + } + minioClient := minioClient{client: mClient} + + resp, pw := io.Pipe() + // Create file async + go func() { + defer pw.Close() + zipw := zip.NewWriter(pw) + defer zipw.Close() + + addToZip := func(name string, modified time.Time) (io.Writer, error) { + f, err := zipw.CreateHeader(&zip.FileHeader{ + Name: name, + NonUTF8: false, + Method: zip.Deflate, + Modified: modified, + }) + return f, err + } + + for _, dObj := range params.ObjectList { + // if a prefix is selected, list and add objects recursively + // the prefixes are not base64 encoded. + if strings.HasSuffix(dObj, "/") { + prefix := dObj + + folders := strings.Split(prefix, "/") + + var folder string + if len(folders) > 1 { + folder = folders[len(folders)-2] + } + + objects, err := listBucketObjects(ListObjectsOpts{ + ctx: ctx, + client: minioClient, + bucketName: params.BucketName, + prefix: prefix, + recursive: true, + withVersions: false, + withMetadata: false, + }) + if err != nil { + pw.CloseWithError(err) + } + + for i, obj := range objects { + name := folder + objects[i].Name[len(prefix)-1:] + + object, err := mClient.GetObject(ctx, params.BucketName, obj.Name, minio.GetObjectOptions{}) + if err != nil { + // Ignore errors, move to next + continue + } + modified, _ := time.Parse(time.RFC3339, obj.LastModified) + + f, err := addToZip(name, modified) + if err != nil { + // Ignore errors, move to next + continue + } + _, err = io.Copy(f, object) + if err != nil { + // We have a partial object, report error. + pw.CloseWithError(err) + return + } + } + + } else { + // add selected individual object + objectData, err := mClient.StatObject(ctx, params.BucketName, dObj, minio.StatObjectOptions{}) + if err != nil { + // Ignore errors, move to next + continue + } + object, err := mClient.GetObject(ctx, params.BucketName, dObj, minio.GetObjectOptions{}) + if err != nil { + // Ignore errors, move to next + continue + } + + f, err := addToZip(dObj, objectData.LastModified) + if err != nil { + // Ignore errors, move to next + continue + } + _, err = io.Copy(f, object) + if err != nil { + // We have a partial object, report error. + pw.CloseWithError(err) + return + } + } + } + }() + + return middleware.ResponderFunc(func(rw http.ResponseWriter, _ runtime.Producer) { + defer resp.Close() + + // indicate it's a download / inline content to the browser, and the size of the object + fileName := "selected_files_" + time.Now().UTC().Format(time.DateTime) + + rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", fileName)) + rw.Header().Set("Content-Type", "application/zip") + + // Copy the stream + _, err := io.Copy(rw, resp) + if err != nil { + ErrorWithContext(ctx, fmt.Errorf("Unable to write all the requested data: %v", err)) + } + }), nil +} + // getDeleteObjectResponse returns whether there was an error on deletion of object func getDeleteObjectResponse(session *models.Principal, params objectApi.DeleteObjectParams) *models.Error { ctx, cancel := context.WithCancel(params.HTTPRequest.Context()) diff --git a/restapi/user_objects_test.go b/restapi/user_objects_test.go index dc9dc5001..d31cdd8c0 100644 --- a/restapi/user_objects_test.go +++ b/restapi/user_objects_test.go @@ -22,11 +22,15 @@ import ( "errors" "fmt" "io" + "net/http" "path/filepath" "reflect" "testing" "time" + "github.com/go-openapi/runtime/middleware" + "github.com/minio/console/restapi/operations/object" + "github.com/go-openapi/swag" "github.com/minio/console/models" mc "github.com/minio/mc/cmd" @@ -905,7 +909,7 @@ func Test_deleteObjects(t *testing.T) { } func Test_shareObject(t *testing.T) { - assert := assert.New(t) + tAssert := assert.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := s3ClientMock{} @@ -978,7 +982,7 @@ func Test_shareObject(t *testing.T) { return } } else { - assert.Equal(*url, tt.expected) + tAssert.Equal(*url, tt.expected) } }) } @@ -1054,7 +1058,7 @@ func Test_putObjectLegalHold(t *testing.T) { } func Test_putObjectRetention(t *testing.T) { - assert := assert.New(t) + tAssert := assert.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := minioClientMock{} @@ -1176,16 +1180,16 @@ func Test_putObjectRetention(t *testing.T) { err := setObjectRetention(ctx, client, tt.args.bucket, tt.args.prefix, tt.args.versionID, tt.args.opts) if tt.wantError != nil { fmt.Println(t.Name()) - assert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("setObjectRetention() error: `%s`, wantErr: `%s`", err, tt.wantError)) + tAssert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("setObjectRetention() error: `%s`, wantErr: `%s`", err, tt.wantError)) } else { - assert.Nil(err, fmt.Sprintf("setObjectRetention() error: %v, wantErr: %v", err, tt.wantError)) + tAssert.Nil(err, fmt.Sprintf("setObjectRetention() error: %v, wantErr: %v", err, tt.wantError)) } }) } } func Test_deleteObjectRetention(t *testing.T) { - assert := assert.New(t) + tAssert := assert.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := minioClientMock{} @@ -1219,16 +1223,16 @@ func Test_deleteObjectRetention(t *testing.T) { err := deleteObjectRetention(ctx, client, tt.args.bucket, tt.args.prefix, tt.args.versionID) if tt.wantError != nil { fmt.Println(t.Name()) - assert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("deleteObjectRetention() error: `%s`, wantErr: `%s`", err, tt.wantError)) + tAssert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("deleteObjectRetention() error: `%s`, wantErr: `%s`", err, tt.wantError)) } else { - assert.Nil(err, fmt.Sprintf("deleteObjectRetention() error: %v, wantErr: %v", err, tt.wantError)) + tAssert.Nil(err, fmt.Sprintf("deleteObjectRetention() error: %v, wantErr: %v", err, tt.wantError)) } }) } } func Test_getObjectInfo(t *testing.T) { - assert := assert.New(t) + tAssert := assert.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() client := minioClientMock{} @@ -1272,9 +1276,9 @@ func Test_getObjectInfo(t *testing.T) { _, err := getObjectInfo(ctx, client, tt.args.bucketName, tt.args.prefix) if tt.wantError != nil { fmt.Println(t.Name()) - assert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("getObjectInfo() error: `%s`, wantErr: `%s`", err, tt.wantError)) + tAssert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("getObjectInfo() error: `%s`, wantErr: `%s`", err, tt.wantError)) } else { - assert.Nil(err, fmt.Sprintf("getObjectInfo() error: %v, wantErr: %v", err, tt.wantError)) + tAssert.Nil(err, fmt.Sprintf("getObjectInfo() error: %v, wantErr: %v", err, tt.wantError)) } }) } @@ -1440,3 +1444,64 @@ func Test_newClientURL(t *testing.T) { }) } } + +func Test_getMultipleFilesDownloadResponse(t *testing.T) { + type args struct { + session *models.Principal + params object.DownloadMultipleObjectsParams + } + + tests := []struct { + name string + args args + want middleware.Responder + want1 *models.Error + }{ + { + name: "test no objects sent for download", + args: args{ + session: nil, + params: object.DownloadMultipleObjectsParams{ + HTTPRequest: &http.Request{}, + BucketName: "test-bucket", + ObjectList: nil, + }, + }, + want: nil, + want1: nil, + }, + { + name: "few objects sent for download", + args: args{ + session: nil, + params: object.DownloadMultipleObjectsParams{ + HTTPRequest: &http.Request{}, + BucketName: "test-bucket", + ObjectList: []string{"test.txt", ",y-obj.doc", "z-obj.png"}, + }, + }, + want: nil, + want1: nil, + }, + { + name: "few prefixes and a file sent for download", + args: args{ + session: nil, + params: object.DownloadMultipleObjectsParams{ + HTTPRequest: &http.Request{}, + BucketName: "test-bucket", + ObjectList: []string{"my-folder/", "my-folder/test-nested", "z-obj.png"}, + }, + }, + want: nil, + want1: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, got1 := getMultipleFilesDownloadResponse(tt.args.session, tt.args.params) + assert.Equal(t, tt.want1, got1) + assert.NotNil(t, got) + }) + } +} diff --git a/swagger.yml b/swagger.yml index e8ba95a42..41b5cb395 100644 --- a/swagger.yml +++ b/swagger.yml @@ -439,6 +439,39 @@ paths: tags: - Object + /buckets/{bucket_name}/objects/download-multiple: + post: + summary: Download Multiple Objects + operationId: DownloadMultipleObjects + security: + - key: [ ] + - anonymous: [ ] + produces: + - application/octet-stream + parameters: + - name: bucket_name + in: path + required: true + type: string + - name: objectList + in: body + required: true + schema: + type: array + items: + type: string + responses: + 200: + description: A successful response. + schema: + type: file + default: + description: Generic error response. + schema: + $ref: "#/definitions/error" + tags: + - Object + /buckets/{bucket_name}/objects/download: get: summary: Download Object