feat: download multiple object selection as zip ignoring any deleted objects selected (#2965)

This commit is contained in:
Prakash Senthil Vel
2023-08-03 05:58:25 +05:30
committed by GitHub
parent d116a35a6d
commit b968cc25ad
14 changed files with 1002 additions and 28 deletions

View File

@@ -17,8 +17,10 @@
package integration
import (
"bytes"
"context"
"encoding/base64"
"encoding/json"
"fmt"
"log"
"math/rand"
@@ -194,3 +196,86 @@ func TestObjectGet(t *testing.T) {
})
}
}
func downloadMultipleFiles(bucketName string, objects []string) (*http.Response, error) {
requestURL := fmt.Sprintf("http://localhost:9090/api/v1/buckets/%s/objects/download-multiple", bucketName)
postReqParams, _ := json.Marshal(objects)
reqBody := bytes.NewReader(postReqParams)
request, err := http.NewRequest(
"POST", requestURL, reqBody)
if err != nil {
log.Println(err)
return nil, nil
}
request.Header.Add("Cookie", fmt.Sprintf("token=%s", token))
request.Header.Add("Content-Type", "application/json")
client := &http.Client{
Timeout: 2 * time.Second,
}
response, err := client.Do(request)
return response, err
}
func TestDownloadMultipleFiles(t *testing.T) {
assert := assert.New(t)
type args struct {
bucketName string
objectLis []string
}
tests := []struct {
name string
args args
expectedStatus int
expectedError bool
}{
{
name: "Test empty Bucket",
args: args{
bucketName: "",
},
expectedStatus: 400,
expectedError: true,
},
{
name: "Test empty object list",
args: args{
bucketName: "test-bucket",
},
expectedStatus: 400,
expectedError: true,
},
{
name: "Test with bucket and object list",
args: args{
bucketName: "test-bucket",
objectLis: []string{
"my-object.txt",
"test-prefix/",
"test-prefix/nested-prefix/",
"test-prefix/nested-prefix/deep-nested/",
},
},
expectedStatus: 200,
expectedError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
resp, err := downloadMultipleFiles(tt.args.bucketName, tt.args.objectLis)
if tt.expectedError {
assert.Nil(err)
if err != nil {
log.Println(err)
return
}
}
if resp != nil {
assert.NotNil(resp)
}
})
}
}

View File

@@ -2142,6 +2142,28 @@ export class Api<
...params,
}),
/**
* No description
*
* @tags Object
* @name DownloadMultipleObjects
* @summary Download Multiple Objects
* @request POST:/buckets/{bucket_name}/objects/download-multiple
* @secure
*/
downloadMultipleObjects: (
bucketName: string,
objectList: string[],
params: RequestParams = {},
) =>
this.request<File, Error>({
path: `/buckets/${bucketName}/objects/download-multiple`,
method: "POST",
body: objectList,
secure: true,
...params,
}),
/**
* No description
*

View File

@@ -912,6 +912,11 @@ const ListObjects = () => {
createdTime = DateTime.fromISO(bucketInfo.creation_date);
}
const downloadToolTip =
selectedObjects?.length <= 1
? "Download Selected"
: ` Download selected objects as Zip. Any Deleted objects in the selection would be skipped from download.`;
const multiActionButtons = [
{
action: () => {
@@ -921,7 +926,7 @@ const ListObjects = () => {
disabled: !canDownload || selectedObjects?.length === 0,
icon: <DownloadIcon />,
tooltip: canDownload
? "Download Selected"
? downloadToolTip
: permissionTooltipHelper(
[IAM_SCOPES.S3_GET_OBJECT, IAM_SCOPES.S3_GET_ACTIONS],
"download objects from this bucket",

View File

@@ -18,8 +18,52 @@ import { BucketObjectItem } from "./ListObjects/types";
import { encodeURLString } from "../../../../../common/utils";
import { removeTrace } from "../../../ObjectBrowser/transferManager";
import store from "../../../../../store";
import { PermissionResource } from "api/consoleApi";
import { ContentType, PermissionResource } from "api/consoleApi";
import { api } from "../../../../../api";
import { setErrorSnackMessage } from "../../../../../systemSlice";
const downloadWithLink = (href: string, downloadFileName: string) => {
const link = document.createElement("a");
link.href = href;
link.download = downloadFileName;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
};
export const downloadSelectedAsZip = async (
bucketName: string,
objectList: string[],
resultFileName: string,
) => {
const state = store.getState();
const anonymousMode = state.system.anonymousMode;
try {
const resp = await api.buckets.downloadMultipleObjects(
bucketName,
objectList,
{
type: ContentType.Json,
headers: anonymousMode
? {
"X-Anonymous": "1",
}
: undefined,
},
);
const blob = await resp.blob();
const href = window.URL.createObjectURL(blob);
downloadWithLink(href, resultFileName);
} catch (err: any) {
store.dispatch(
setErrorSnackMessage({
errorMessage: `Download of multiple files failed. ${err.statusText}`,
detailedError: "",
}),
);
}
};
export const download = (
bucketName: string,
objectPath: string,
@@ -33,8 +77,6 @@ export const download = (
abortCallback: () => void,
toastCallback: () => void,
) => {
const anchor = document.createElement("a");
document.body.appendChild(anchor);
let basename = document.baseURI.replace(window.location.origin, "");
const state = store.getState();
const anonymousMode = state.system.anonymousMode;
@@ -90,12 +132,7 @@ export const download = (
removeTrace(id);
var link = document.createElement("a");
link.href = window.URL.createObjectURL(req.response);
link.download = filename;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
downloadWithLink(window.URL.createObjectURL(req.response), filename);
} else {
if (req.getResponseHeader("Content-Type") === "application/json") {
const rspBody: { detailedMessage?: string } = JSON.parse(

View File

@@ -19,7 +19,10 @@ import { AppState } from "../../../store";
import { encodeURLString, getClientOS } from "../../../common/utils";
import { BucketObjectItem } from "../Buckets/ListBuckets/Objects/ListObjects/types";
import { makeid, storeCallForObjectWithID } from "./transferManager";
import { download } from "../Buckets/ListBuckets/Objects/utils";
import {
download,
downloadSelectedAsZip,
} from "../Buckets/ListBuckets/Objects/utils";
import {
cancelObjectInList,
completeObject,
@@ -33,6 +36,7 @@ import {
updateProgress,
} from "./objectBrowserSlice";
import { setSnackBarMessage } from "../../../systemSlice";
import { DateTime } from "luxon";
export const downloadSelected = createAsyncThunk(
"objectBrowser/downloadSelected",
@@ -104,8 +108,7 @@ export const downloadSelected = createAsyncThunk(
itemsToDownload = state.objectBrowser.records.filter(filterFunction);
// I case just one element is selected, then we trigger download modal validation.
// We are going to enforce zip download when multiple files are selected
// In case just one element is selected, then we trigger download modal validation.
if (itemsToDownload.length === 1) {
if (
itemsToDownload[0].name.length > 200 &&
@@ -113,12 +116,34 @@ export const downloadSelected = createAsyncThunk(
) {
dispatch(setDownloadRenameModal(itemsToDownload[0]));
return;
} else {
downloadObject(itemsToDownload[0]);
}
} else {
if (itemsToDownload.length === 1) {
downloadObject(itemsToDownload[0]);
} else if (itemsToDownload.length > 1) {
const fileName = `${DateTime.now().toFormat(
"LL-dd-yyyy-HH-mm-ss",
)}_files_list.zip`;
// We are enforcing zip download when multiple files are selected for better user experience
const multiObjList = itemsToDownload.reduce((dwList: any[], bi) => {
// Download objects/prefixes(recursively) as zip
// Skip any deleted files selected via "Show deleted objects" in selection and log for debugging
const isDeleted = bi?.delete_flag;
if (bi && !isDeleted) {
dwList.push(bi.name);
} else {
console.log(`Skipping ${bi?.name} from download.`);
}
return dwList;
}, []);
await downloadSelectedAsZip(bucketName, multiObjList, fileName);
return;
}
}
itemsToDownload.forEach((filteredItem) => {
downloadObject(filteredItem);
});
}
},
);

View File

@@ -1646,6 +1646,59 @@ func init() {
}
}
},
"/buckets/{bucket_name}/objects/download-multiple": {
"post": {
"security": [
{
"key": []
},
{
"anonymous": []
}
],
"produces": [
"application/octet-stream"
],
"tags": [
"Object"
],
"summary": "Download Multiple Objects",
"operationId": "DownloadMultipleObjects",
"parameters": [
{
"type": "string",
"name": "bucket_name",
"in": "path",
"required": true
},
{
"name": "objectList",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
],
"responses": {
"200": {
"description": "A successful response.",
"schema": {
"type": "file"
}
},
"default": {
"description": "Generic error response.",
"schema": {
"$ref": "#/definitions/error"
}
}
}
}
},
"/buckets/{bucket_name}/objects/legalhold": {
"put": {
"tags": [
@@ -10660,6 +10713,59 @@ func init() {
}
}
},
"/buckets/{bucket_name}/objects/download-multiple": {
"post": {
"security": [
{
"key": []
},
{
"anonymous": []
}
],
"produces": [
"application/octet-stream"
],
"tags": [
"Object"
],
"summary": "Download Multiple Objects",
"operationId": "DownloadMultipleObjects",
"parameters": [
{
"type": "string",
"name": "bucket_name",
"in": "path",
"required": true
},
{
"name": "objectList",
"in": "body",
"required": true,
"schema": {
"type": "array",
"items": {
"type": "string"
}
}
}
],
"responses": {
"200": {
"description": "A successful response.",
"schema": {
"type": "file"
}
},
"default": {
"description": "Generic error response.",
"schema": {
"$ref": "#/definitions/error"
}
}
}
}
},
"/buckets/{bucket_name}/objects/legalhold": {
"put": {
"tags": [

View File

@@ -211,6 +211,9 @@ func NewConsoleAPI(spec *loads.Document) *ConsoleAPI {
ObjectDownloadObjectHandler: object.DownloadObjectHandlerFunc(func(params object.DownloadObjectParams, principal *models.Principal) middleware.Responder {
return middleware.NotImplemented("operation object.DownloadObject has not yet been implemented")
}),
ObjectDownloadMultipleObjectsHandler: object.DownloadMultipleObjectsHandlerFunc(func(params object.DownloadMultipleObjectsParams, principal *models.Principal) middleware.Responder {
return middleware.NotImplemented("operation object.DownloadMultipleObjects has not yet been implemented")
}),
TieringEditTierCredentialsHandler: tiering.EditTierCredentialsHandlerFunc(func(params tiering.EditTierCredentialsParams, principal *models.Principal) middleware.Responder {
return middleware.NotImplemented("operation tiering.EditTierCredentials has not yet been implemented")
}),
@@ -701,6 +704,8 @@ type ConsoleAPI struct {
BucketDisableBucketEncryptionHandler bucket.DisableBucketEncryptionHandler
// ObjectDownloadObjectHandler sets the operation handler for the download object operation
ObjectDownloadObjectHandler object.DownloadObjectHandler
// ObjectDownloadMultipleObjectsHandler sets the operation handler for the download multiple objects operation
ObjectDownloadMultipleObjectsHandler object.DownloadMultipleObjectsHandler
// TieringEditTierCredentialsHandler sets the operation handler for the edit tier credentials operation
TieringEditTierCredentialsHandler tiering.EditTierCredentialsHandler
// BucketEnableBucketEncryptionHandler sets the operation handler for the enable bucket encryption operation
@@ -1145,6 +1150,9 @@ func (o *ConsoleAPI) Validate() error {
if o.ObjectDownloadObjectHandler == nil {
unregistered = append(unregistered, "object.DownloadObjectHandler")
}
if o.ObjectDownloadMultipleObjectsHandler == nil {
unregistered = append(unregistered, "object.DownloadMultipleObjectsHandler")
}
if o.TieringEditTierCredentialsHandler == nil {
unregistered = append(unregistered, "tiering.EditTierCredentialsHandler")
}
@@ -1761,6 +1769,10 @@ func (o *ConsoleAPI) initHandlerCache() {
o.handlers["GET"] = make(map[string]http.Handler)
}
o.handlers["GET"]["/buckets/{bucket_name}/objects/download"] = object.NewDownloadObject(o.context, o.ObjectDownloadObjectHandler)
if o.handlers["POST"] == nil {
o.handlers["POST"] = make(map[string]http.Handler)
}
o.handlers["POST"]["/buckets/{bucket_name}/objects/download-multiple"] = object.NewDownloadMultipleObjects(o.context, o.ObjectDownloadMultipleObjectsHandler)
if o.handlers["PUT"] == nil {
o.handlers["PUT"] = make(map[string]http.Handler)
}

View File

@@ -0,0 +1,88 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2023 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package object
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the generate command
import (
"net/http"
"github.com/go-openapi/runtime/middleware"
"github.com/minio/console/models"
)
// DownloadMultipleObjectsHandlerFunc turns a function with the right signature into a download multiple objects handler
type DownloadMultipleObjectsHandlerFunc func(DownloadMultipleObjectsParams, *models.Principal) middleware.Responder
// Handle executing the request and returning a response
func (fn DownloadMultipleObjectsHandlerFunc) Handle(params DownloadMultipleObjectsParams, principal *models.Principal) middleware.Responder {
return fn(params, principal)
}
// DownloadMultipleObjectsHandler interface for that can handle valid download multiple objects params
type DownloadMultipleObjectsHandler interface {
Handle(DownloadMultipleObjectsParams, *models.Principal) middleware.Responder
}
// NewDownloadMultipleObjects creates a new http.Handler for the download multiple objects operation
func NewDownloadMultipleObjects(ctx *middleware.Context, handler DownloadMultipleObjectsHandler) *DownloadMultipleObjects {
return &DownloadMultipleObjects{Context: ctx, Handler: handler}
}
/*
DownloadMultipleObjects swagger:route POST /buckets/{bucket_name}/objects/download-multiple Object downloadMultipleObjects
Download Multiple Objects
*/
type DownloadMultipleObjects struct {
Context *middleware.Context
Handler DownloadMultipleObjectsHandler
}
func (o *DownloadMultipleObjects) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
route, rCtx, _ := o.Context.RouteInfo(r)
if rCtx != nil {
*r = *rCtx
}
var Params = NewDownloadMultipleObjectsParams()
uprinc, aCtx, err := o.Context.Authorize(r, route)
if err != nil {
o.Context.Respond(rw, r, route.Produces, route, err)
return
}
if aCtx != nil {
*r = *aCtx
}
var principal *models.Principal
if uprinc != nil {
principal = uprinc.(*models.Principal) // this is really a models.Principal, I promise
}
if err := o.Context.BindValidRequest(r, route, &Params); err != nil { // bind params
o.Context.Respond(rw, r, route.Produces, route, err)
return
}
res := o.Handler.Handle(Params, principal) // actually handle the request
o.Context.Respond(rw, r, route.Produces, route, res)
}

View File

@@ -0,0 +1,112 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2023 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package object
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the swagger generate command
import (
"io"
"net/http"
"github.com/go-openapi/errors"
"github.com/go-openapi/runtime"
"github.com/go-openapi/runtime/middleware"
"github.com/go-openapi/strfmt"
)
// NewDownloadMultipleObjectsParams creates a new DownloadMultipleObjectsParams object
//
// There are no default values defined in the spec.
func NewDownloadMultipleObjectsParams() DownloadMultipleObjectsParams {
return DownloadMultipleObjectsParams{}
}
// DownloadMultipleObjectsParams contains all the bound params for the download multiple objects operation
// typically these are obtained from a http.Request
//
// swagger:parameters DownloadMultipleObjects
type DownloadMultipleObjectsParams struct {
// HTTP Request Object
HTTPRequest *http.Request `json:"-"`
/*
Required: true
In: path
*/
BucketName string
/*
Required: true
In: body
*/
ObjectList []string
}
// BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface
// for simple values it will use straight method calls.
//
// To ensure default values, the struct must have been initialized with NewDownloadMultipleObjectsParams() beforehand.
func (o *DownloadMultipleObjectsParams) BindRequest(r *http.Request, route *middleware.MatchedRoute) error {
var res []error
o.HTTPRequest = r
rBucketName, rhkBucketName, _ := route.Params.GetOK("bucket_name")
if err := o.bindBucketName(rBucketName, rhkBucketName, route.Formats); err != nil {
res = append(res, err)
}
if runtime.HasBody(r) {
defer r.Body.Close()
var body []string
if err := route.Consumer.Consume(r.Body, &body); err != nil {
if err == io.EOF {
res = append(res, errors.Required("objectList", "body", ""))
} else {
res = append(res, errors.NewParseError("objectList", "body", "", err))
}
} else {
// no validation required on inline body
o.ObjectList = body
}
} else {
res = append(res, errors.Required("objectList", "body", ""))
}
if len(res) > 0 {
return errors.CompositeValidationError(res...)
}
return nil
}
// bindBucketName binds and validates parameter BucketName from path.
func (o *DownloadMultipleObjectsParams) bindBucketName(rawData []string, hasKey bool, formats strfmt.Registry) error {
var raw string
if len(rawData) > 0 {
raw = rawData[len(rawData)-1]
}
// Required: true
// Parameter is provided by construction from the route
o.BucketName = raw
return nil
}

View File

@@ -0,0 +1,134 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2023 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package object
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the swagger generate command
import (
"io"
"net/http"
"github.com/go-openapi/runtime"
"github.com/minio/console/models"
)
// DownloadMultipleObjectsOKCode is the HTTP code returned for type DownloadMultipleObjectsOK
const DownloadMultipleObjectsOKCode int = 200
/*
DownloadMultipleObjectsOK A successful response.
swagger:response downloadMultipleObjectsOK
*/
type DownloadMultipleObjectsOK struct {
/*
In: Body
*/
Payload io.ReadCloser `json:"body,omitempty"`
}
// NewDownloadMultipleObjectsOK creates DownloadMultipleObjectsOK with default headers values
func NewDownloadMultipleObjectsOK() *DownloadMultipleObjectsOK {
return &DownloadMultipleObjectsOK{}
}
// WithPayload adds the payload to the download multiple objects o k response
func (o *DownloadMultipleObjectsOK) WithPayload(payload io.ReadCloser) *DownloadMultipleObjectsOK {
o.Payload = payload
return o
}
// SetPayload sets the payload to the download multiple objects o k response
func (o *DownloadMultipleObjectsOK) SetPayload(payload io.ReadCloser) {
o.Payload = payload
}
// WriteResponse to the client
func (o *DownloadMultipleObjectsOK) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) {
rw.WriteHeader(200)
payload := o.Payload
if err := producer.Produce(rw, payload); err != nil {
panic(err) // let the recovery middleware deal with this
}
}
/*
DownloadMultipleObjectsDefault Generic error response.
swagger:response downloadMultipleObjectsDefault
*/
type DownloadMultipleObjectsDefault struct {
_statusCode int
/*
In: Body
*/
Payload *models.Error `json:"body,omitempty"`
}
// NewDownloadMultipleObjectsDefault creates DownloadMultipleObjectsDefault with default headers values
func NewDownloadMultipleObjectsDefault(code int) *DownloadMultipleObjectsDefault {
if code <= 0 {
code = 500
}
return &DownloadMultipleObjectsDefault{
_statusCode: code,
}
}
// WithStatusCode adds the status to the download multiple objects default response
func (o *DownloadMultipleObjectsDefault) WithStatusCode(code int) *DownloadMultipleObjectsDefault {
o._statusCode = code
return o
}
// SetStatusCode sets the status to the download multiple objects default response
func (o *DownloadMultipleObjectsDefault) SetStatusCode(code int) {
o._statusCode = code
}
// WithPayload adds the payload to the download multiple objects default response
func (o *DownloadMultipleObjectsDefault) WithPayload(payload *models.Error) *DownloadMultipleObjectsDefault {
o.Payload = payload
return o
}
// SetPayload sets the payload to the download multiple objects default response
func (o *DownloadMultipleObjectsDefault) SetPayload(payload *models.Error) {
o.Payload = payload
}
// WriteResponse to the client
func (o *DownloadMultipleObjectsDefault) WriteResponse(rw http.ResponseWriter, producer runtime.Producer) {
rw.WriteHeader(o._statusCode)
if o.Payload != nil {
payload := o.Payload
if err := producer.Produce(rw, payload); err != nil {
panic(err) // let the recovery middleware deal with this
}
}
}

View File

@@ -0,0 +1,116 @@
// Code generated by go-swagger; DO NOT EDIT.
// This file is part of MinIO Console Server
// Copyright (c) 2023 MinIO, Inc.
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
//
package object
// This file was generated by the swagger tool.
// Editing this file might prove futile when you re-run the generate command
import (
"errors"
"net/url"
golangswaggerpaths "path"
"strings"
)
// DownloadMultipleObjectsURL generates an URL for the download multiple objects operation
type DownloadMultipleObjectsURL struct {
BucketName string
_basePath string
// avoid unkeyed usage
_ struct{}
}
// WithBasePath sets the base path for this url builder, only required when it's different from the
// base path specified in the swagger spec.
// When the value of the base path is an empty string
func (o *DownloadMultipleObjectsURL) WithBasePath(bp string) *DownloadMultipleObjectsURL {
o.SetBasePath(bp)
return o
}
// SetBasePath sets the base path for this url builder, only required when it's different from the
// base path specified in the swagger spec.
// When the value of the base path is an empty string
func (o *DownloadMultipleObjectsURL) SetBasePath(bp string) {
o._basePath = bp
}
// Build a url path and query string
func (o *DownloadMultipleObjectsURL) Build() (*url.URL, error) {
var _result url.URL
var _path = "/buckets/{bucket_name}/objects/download-multiple"
bucketName := o.BucketName
if bucketName != "" {
_path = strings.Replace(_path, "{bucket_name}", bucketName, -1)
} else {
return nil, errors.New("bucketName is required on DownloadMultipleObjectsURL")
}
_basePath := o._basePath
if _basePath == "" {
_basePath = "/api/v1"
}
_result.Path = golangswaggerpaths.Join(_basePath, _path)
return &_result, nil
}
// Must is a helper function to panic when the url builder returns an error
func (o *DownloadMultipleObjectsURL) Must(u *url.URL, err error) *url.URL {
if err != nil {
panic(err)
}
if u == nil {
panic("url can't be nil")
}
return u
}
// String returns the string representation of the path with query string
func (o *DownloadMultipleObjectsURL) String() string {
return o.Must(o.Build()).String()
}
// BuildFull builds a full url with scheme, host, path and query string
func (o *DownloadMultipleObjectsURL) BuildFull(scheme, host string) (*url.URL, error) {
if scheme == "" {
return nil, errors.New("scheme is required for a full url on DownloadMultipleObjectsURL")
}
if host == "" {
return nil, errors.New("host is required for a full url on DownloadMultipleObjectsURL")
}
base, err := o.Build()
if err != nil {
return nil, err
}
base.Scheme = scheme
base.Host = host
return base, nil
}
// StringFull returns the string representation of a complete url
func (o *DownloadMultipleObjectsURL) StringFull(scheme, host string) string {
return o.Must(o.BuildFull(scheme, host)).String()
}

View File

@@ -107,6 +107,21 @@ func registerObjectsHandlers(api *operations.ConsoleAPI) {
}
return resp
})
// download multiple objects
api.ObjectDownloadMultipleObjectsHandler = objectApi.DownloadMultipleObjectsHandlerFunc(func(params objectApi.DownloadMultipleObjectsParams, session *models.Principal) middleware.Responder {
ctx := params.HTTPRequest.Context()
if len(params.ObjectList) < 1 {
return objectApi.NewDownloadMultipleObjectsDefault(400).WithPayload(ErrorWithContext(ctx, errors.New("could not download, since object list is empty")))
}
var resp middleware.Responder
var err *models.Error
resp, err = getMultipleFilesDownloadResponse(session, params)
if err != nil {
return objectApi.NewDownloadMultipleObjectsDefault(int(err.Code)).WithPayload(err)
}
return resp
})
// upload object
api.ObjectPostBucketsBucketNameObjectsUploadHandler = objectApi.PostBucketsBucketNameObjectsUploadHandlerFunc(func(params objectApi.PostBucketsBucketNameObjectsUploadParams, session *models.Principal) middleware.Responder {
if err := getUploadObjectResponse(session, params); err != nil {
@@ -620,6 +635,125 @@ func getDownloadFolderResponse(session *models.Principal, params objectApi.Downl
}), nil
}
func getMultipleFilesDownloadResponse(session *models.Principal, params objectApi.DownloadMultipleObjectsParams) (middleware.Responder, *models.Error) {
ctx := params.HTTPRequest.Context()
mClient, err := newMinioClient(session, getClientIP(params.HTTPRequest))
if err != nil {
return nil, ErrorWithContext(ctx, err)
}
minioClient := minioClient{client: mClient}
resp, pw := io.Pipe()
// Create file async
go func() {
defer pw.Close()
zipw := zip.NewWriter(pw)
defer zipw.Close()
addToZip := func(name string, modified time.Time) (io.Writer, error) {
f, err := zipw.CreateHeader(&zip.FileHeader{
Name: name,
NonUTF8: false,
Method: zip.Deflate,
Modified: modified,
})
return f, err
}
for _, dObj := range params.ObjectList {
// if a prefix is selected, list and add objects recursively
// the prefixes are not base64 encoded.
if strings.HasSuffix(dObj, "/") {
prefix := dObj
folders := strings.Split(prefix, "/")
var folder string
if len(folders) > 1 {
folder = folders[len(folders)-2]
}
objects, err := listBucketObjects(ListObjectsOpts{
ctx: ctx,
client: minioClient,
bucketName: params.BucketName,
prefix: prefix,
recursive: true,
withVersions: false,
withMetadata: false,
})
if err != nil {
pw.CloseWithError(err)
}
for i, obj := range objects {
name := folder + objects[i].Name[len(prefix)-1:]
object, err := mClient.GetObject(ctx, params.BucketName, obj.Name, minio.GetObjectOptions{})
if err != nil {
// Ignore errors, move to next
continue
}
modified, _ := time.Parse(time.RFC3339, obj.LastModified)
f, err := addToZip(name, modified)
if err != nil {
// Ignore errors, move to next
continue
}
_, err = io.Copy(f, object)
if err != nil {
// We have a partial object, report error.
pw.CloseWithError(err)
return
}
}
} else {
// add selected individual object
objectData, err := mClient.StatObject(ctx, params.BucketName, dObj, minio.StatObjectOptions{})
if err != nil {
// Ignore errors, move to next
continue
}
object, err := mClient.GetObject(ctx, params.BucketName, dObj, minio.GetObjectOptions{})
if err != nil {
// Ignore errors, move to next
continue
}
f, err := addToZip(dObj, objectData.LastModified)
if err != nil {
// Ignore errors, move to next
continue
}
_, err = io.Copy(f, object)
if err != nil {
// We have a partial object, report error.
pw.CloseWithError(err)
return
}
}
}
}()
return middleware.ResponderFunc(func(rw http.ResponseWriter, _ runtime.Producer) {
defer resp.Close()
// indicate it's a download / inline content to the browser, and the size of the object
fileName := "selected_files_" + time.Now().UTC().Format(time.DateTime)
rw.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s.zip\"", fileName))
rw.Header().Set("Content-Type", "application/zip")
// Copy the stream
_, err := io.Copy(rw, resp)
if err != nil {
ErrorWithContext(ctx, fmt.Errorf("Unable to write all the requested data: %v", err))
}
}), nil
}
// getDeleteObjectResponse returns whether there was an error on deletion of object
func getDeleteObjectResponse(session *models.Principal, params objectApi.DeleteObjectParams) *models.Error {
ctx, cancel := context.WithCancel(params.HTTPRequest.Context())

View File

@@ -22,11 +22,15 @@ import (
"errors"
"fmt"
"io"
"net/http"
"path/filepath"
"reflect"
"testing"
"time"
"github.com/go-openapi/runtime/middleware"
"github.com/minio/console/restapi/operations/object"
"github.com/go-openapi/swag"
"github.com/minio/console/models"
mc "github.com/minio/mc/cmd"
@@ -905,7 +909,7 @@ func Test_deleteObjects(t *testing.T) {
}
func Test_shareObject(t *testing.T) {
assert := assert.New(t)
tAssert := assert.New(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := s3ClientMock{}
@@ -978,7 +982,7 @@ func Test_shareObject(t *testing.T) {
return
}
} else {
assert.Equal(*url, tt.expected)
tAssert.Equal(*url, tt.expected)
}
})
}
@@ -1054,7 +1058,7 @@ func Test_putObjectLegalHold(t *testing.T) {
}
func Test_putObjectRetention(t *testing.T) {
assert := assert.New(t)
tAssert := assert.New(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := minioClientMock{}
@@ -1176,16 +1180,16 @@ func Test_putObjectRetention(t *testing.T) {
err := setObjectRetention(ctx, client, tt.args.bucket, tt.args.prefix, tt.args.versionID, tt.args.opts)
if tt.wantError != nil {
fmt.Println(t.Name())
assert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("setObjectRetention() error: `%s`, wantErr: `%s`", err, tt.wantError))
tAssert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("setObjectRetention() error: `%s`, wantErr: `%s`", err, tt.wantError))
} else {
assert.Nil(err, fmt.Sprintf("setObjectRetention() error: %v, wantErr: %v", err, tt.wantError))
tAssert.Nil(err, fmt.Sprintf("setObjectRetention() error: %v, wantErr: %v", err, tt.wantError))
}
})
}
}
func Test_deleteObjectRetention(t *testing.T) {
assert := assert.New(t)
tAssert := assert.New(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := minioClientMock{}
@@ -1219,16 +1223,16 @@ func Test_deleteObjectRetention(t *testing.T) {
err := deleteObjectRetention(ctx, client, tt.args.bucket, tt.args.prefix, tt.args.versionID)
if tt.wantError != nil {
fmt.Println(t.Name())
assert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("deleteObjectRetention() error: `%s`, wantErr: `%s`", err, tt.wantError))
tAssert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("deleteObjectRetention() error: `%s`, wantErr: `%s`", err, tt.wantError))
} else {
assert.Nil(err, fmt.Sprintf("deleteObjectRetention() error: %v, wantErr: %v", err, tt.wantError))
tAssert.Nil(err, fmt.Sprintf("deleteObjectRetention() error: %v, wantErr: %v", err, tt.wantError))
}
})
}
}
func Test_getObjectInfo(t *testing.T) {
assert := assert.New(t)
tAssert := assert.New(t)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
client := minioClientMock{}
@@ -1272,9 +1276,9 @@ func Test_getObjectInfo(t *testing.T) {
_, err := getObjectInfo(ctx, client, tt.args.bucketName, tt.args.prefix)
if tt.wantError != nil {
fmt.Println(t.Name())
assert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("getObjectInfo() error: `%s`, wantErr: `%s`", err, tt.wantError))
tAssert.Equal(tt.wantError.Error(), err.Error(), fmt.Sprintf("getObjectInfo() error: `%s`, wantErr: `%s`", err, tt.wantError))
} else {
assert.Nil(err, fmt.Sprintf("getObjectInfo() error: %v, wantErr: %v", err, tt.wantError))
tAssert.Nil(err, fmt.Sprintf("getObjectInfo() error: %v, wantErr: %v", err, tt.wantError))
}
})
}
@@ -1440,3 +1444,64 @@ func Test_newClientURL(t *testing.T) {
})
}
}
func Test_getMultipleFilesDownloadResponse(t *testing.T) {
type args struct {
session *models.Principal
params object.DownloadMultipleObjectsParams
}
tests := []struct {
name string
args args
want middleware.Responder
want1 *models.Error
}{
{
name: "test no objects sent for download",
args: args{
session: nil,
params: object.DownloadMultipleObjectsParams{
HTTPRequest: &http.Request{},
BucketName: "test-bucket",
ObjectList: nil,
},
},
want: nil,
want1: nil,
},
{
name: "few objects sent for download",
args: args{
session: nil,
params: object.DownloadMultipleObjectsParams{
HTTPRequest: &http.Request{},
BucketName: "test-bucket",
ObjectList: []string{"test.txt", ",y-obj.doc", "z-obj.png"},
},
},
want: nil,
want1: nil,
},
{
name: "few prefixes and a file sent for download",
args: args{
session: nil,
params: object.DownloadMultipleObjectsParams{
HTTPRequest: &http.Request{},
BucketName: "test-bucket",
ObjectList: []string{"my-folder/", "my-folder/test-nested", "z-obj.png"},
},
},
want: nil,
want1: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, got1 := getMultipleFilesDownloadResponse(tt.args.session, tt.args.params)
assert.Equal(t, tt.want1, got1)
assert.NotNil(t, got)
})
}
}

View File

@@ -439,6 +439,39 @@ paths:
tags:
- Object
/buckets/{bucket_name}/objects/download-multiple:
post:
summary: Download Multiple Objects
operationId: DownloadMultipleObjects
security:
- key: [ ]
- anonymous: [ ]
produces:
- application/octet-stream
parameters:
- name: bucket_name
in: path
required: true
type: string
- name: objectList
in: body
required: true
schema:
type: array
items:
type: string
responses:
200:
description: A successful response.
schema:
type: file
default:
description: Generic error response.
schema:
$ref: "#/definitions/error"
tags:
- Object
/buckets/{bucket_name}/objects/download:
get:
summary: Download Object