Allow arbitrary number of file uploads (#554)

Parameter definition for file upload on swagger.yaml was removed
since go-swagger doesn't support multiple upload of files. Implementation
was done instead on  user_objects.go file.

Co-authored-by: Daniel Valdivia <hola@danielvaldivia.com>

portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx

@@ -33,7 +33,6 @@ import {
   searchField,
 } from "../../../../Common/FormComponents/common/styleLibrary";
 import PageHeader from "../../../../Common/PageHeader/PageHeader";
-import { isNullOrUndefined } from "util";
 import { Button, Input } from "@material-ui/core";
 import * as reactMoment from "react-moment";
 import { CreateIcon } from "../../../../../../icons";
@@ -256,16 +255,28 @@ const ListObjects = ({
   };
 
   const upload = (e: any, bucketName: string, path: string) => {
-    if (isNullOrUndefined(e) || isNullOrUndefined(e.target)) {
+    if (
+      e === null ||
+      e === undefined ||
+      e.target === null ||
+      e.target === undefined
+    ) {
       return;
     }
     e.preventDefault();
-    let file = e.target.files[0];
-    const fileName = file.name;
-
-    const objectName = `${path}${fileName}`;
-    let uploadUrl = `/api/v1/buckets/${bucketName}/objects/upload?prefix=${objectName}`;
+    let files = e.target.files;
+    let uploadUrl = `/api/v1/buckets/${bucketName}/objects/upload`;
+    if (path !== "") {
+      uploadUrl = `${uploadUrl}?prefix=${path}`;
+    }
     let xhr = new XMLHttpRequest();
+    const areMultipleFiles = files.length > 1 ? true : false;
+    const errorMessage = `An error occurred while uploading the file${
+      areMultipleFiles ? "s" : ""
+    }.`;
+    const okMessage = `Object${
+      areMultipleFiles ? "s" : ``
+    } uploaded successfully.`;
 
     xhr.open("POST", uploadUrl, true);
 
@@ -277,15 +288,15 @@ const ListObjects = ({
         xhr.status === 400 ||
         xhr.status === 500
       ) {
-        setSnackBarMessage("An error occurred while uploading the file.");
+        setSnackBarMessage(errorMessage);
       }
       if (xhr.status === 200) {
-        setSnackBarMessage("Object uploaded successfully.");
+        setSnackBarMessage(okMessage);
       }
     };
 
     xhr.upload.addEventListener("error", (event) => {
-      setSnackBarMessage("An error occurred while uploading the file.");
+      setSnackBarMessage(errorMessage);
     });
 
     xhr.upload.addEventListener("progress", (event) => {
@@ -293,7 +304,7 @@ const ListObjects = ({
     });
 
     xhr.onerror = () => {
-      setSnackBarMessage("An error occurred while uploading the file.");
+      setSnackBarMessage(errorMessage);
     };
     xhr.onloadend = () => {
       setLoading(true);
@@ -301,9 +312,13 @@ const ListObjects = ({
     };
 
     const formData = new FormData();
-    const blobFile = new Blob([file]);
 
-    formData.append("upfile", blobFile);
+    for (let file of files) {
+      const fileName = file.name;
+      const blobFile = new Blob([file]);
+      formData.append(fileName, blobFile);
+    }
+
     xhr.send(formData);
     e.target.value = null;
   };
@@ -454,6 +469,7 @@ const ListObjects = ({
                 File
                 <Input
                   type="file"
+                  inputProps={{ multiple: true }}
                   onChange={(e) => uploadObject(e)}
                   id="file-input"
                   style={{ display: "none" }}

restapi/embedded_spec.go

@@ -860,12 +860,6 @@ func init() {
         ],
         "summary": "Uploads an Object.",
         "parameters": [
-          {
-            "type": "file",
-            "name": "upfile",
-            "in": "formData",
-            "required": true
-          },
           {
             "type": "string",
             "name": "bucket_name",
@@ -875,8 +869,7 @@ func init() {
           {
             "type": "string",
             "name": "prefix",
-            "in": "query",
-            "required": true
+            "in": "query"
           }
         ],
         "responses": {
@@ -6333,12 +6326,6 @@ func init() {
         ],
         "summary": "Uploads an Object.",
         "parameters": [
-          {
-            "type": "file",
-            "name": "upfile",
-            "in": "formData",
-            "required": true
-          },
           {
             "type": "string",
             "name": "bucket_name",
@@ -6348,8 +6335,7 @@ func init() {
           {
             "type": "string",
             "name": "prefix",
-            "in": "query",
-            "required": true
+            "in": "query"
           }
         ],
         "responses": {

restapi/operations/user_api/post_buckets_bucket_name_objects_upload_parameters.go

@@ -23,15 +23,12 @@ package user_api
 // Editing this file might prove futile when you re-run the swagger generate command
 
 import (
-	"io"
-	"mime/multipart"
 	"net/http"
 
 	"github.com/go-openapi/errors"
 	"github.com/go-openapi/runtime"
 	"github.com/go-openapi/runtime/middleware"
 	"github.com/go-openapi/strfmt"
-	"github.com/go-openapi/validate"
 )
 
 // NewPostBucketsBucketNameObjectsUploadParams creates a new PostBucketsBucketNameObjectsUploadParams object
@@ -56,15 +53,9 @@ type PostBucketsBucketNameObjectsUploadParams struct {
 	*/
 	BucketName string
 	/*
-	  Required: true
 	  In: query
 	*/
-	Prefix string
-	/*
-	  Required: true
-	  In: formData
-	*/
-	Upfile io.ReadCloser
+	Prefix *string
 }
 
 // BindRequest both binds and validates a request, it assumes that complex things implement a Validatable(strfmt.Registry) error interface
@@ -78,14 +69,6 @@ func (o *PostBucketsBucketNameObjectsUploadParams) BindRequest(r *http.Request,
 
 	qs := runtime.Values(r.URL.Query())
 
-	if err := r.ParseMultipartForm(32 << 20); err != nil {
-		if err != http.ErrNotMultipart {
-			return errors.New(400, "%v", err)
-		} else if err := r.ParseForm(); err != nil {
-			return errors.New(400, "%v", err)
-		}
-	}
-
 	rBucketName, rhkBucketName, _ := route.Params.GetOK("bucket_name")
 	if err := o.bindBucketName(rBucketName, rhkBucketName, route.Formats); err != nil {
 		res = append(res, err)
@@ -96,16 +79,6 @@ func (o *PostBucketsBucketNameObjectsUploadParams) BindRequest(r *http.Request,
 		res = append(res, err)
 	}
 
-	upfile, upfileHeader, err := r.FormFile("upfile")
-	if err != nil {
-		res = append(res, errors.New(400, "reading file %q failed: %v", "upfile", err))
-	} else if err := o.bindUpfile(upfile, upfileHeader); err != nil {
-		// Required: true
-		res = append(res, err)
-	} else {
-		o.Upfile = &runtime.File{Data: upfile, Header: upfileHeader}
-	}
-
 	if len(res) > 0 {
 		return errors.CompositeValidationError(res...)
 	}
@@ -129,28 +102,18 @@ func (o *PostBucketsBucketNameObjectsUploadParams) bindBucketName(rawData []stri
 
 // bindPrefix binds and validates parameter Prefix from query.
 func (o *PostBucketsBucketNameObjectsUploadParams) bindPrefix(rawData []string, hasKey bool, formats strfmt.Registry) error {
-	if !hasKey {
-		return errors.Required("prefix", "query", rawData)
-	}
 	var raw string
 	if len(rawData) > 0 {
 		raw = rawData[len(rawData)-1]
 	}
 
-	// Required: true
+	// Required: false
 	// AllowEmptyValue: false
-	if err := validate.RequiredString("prefix", "query", raw); err != nil {
-		return err
+	if raw == "" { // empty values pass all other validations
+		return nil
 	}
 
-	o.Prefix = raw
+	o.Prefix = &raw
 
 	return nil
 }
-
-// bindUpfile binds file parameter Upfile.
-//
-// The only supported validations on files are MinLength and MaxLength
-func (o *PostBucketsBucketNameObjectsUploadParams) bindUpfile(file multipart.File, header *multipart.FileHeader) error {
-	return nil
-}

restapi/operations/user_api/post_buckets_bucket_name_objects_upload_urlbuilder.go

@@ -33,7 +33,7 @@ import (
 type PostBucketsBucketNameObjectsUploadURL struct {
 	BucketName string
 
-	Prefix string
+	Prefix *string
 
 	_basePath string
 	// avoid unkeyed usage
@@ -76,7 +76,10 @@ func (o *PostBucketsBucketNameObjectsUploadURL) Build() (*url.URL, error) {
 
 	qs := make(url.Values)
 
-	prefixQ := o.Prefix
+	var prefixQ string
+	if o.Prefix != nil {
+		prefixQ = *o.Prefix
+	}
 	if prefixQ != "" {
 		qs.Set("prefix", prefixQ)
 	}

restapi/user_objects.go

@@ -380,29 +380,86 @@ func getUploadObjectResponse(session *models.Principal, params user_api.PostBuck
 	if err != nil {
 		return prepareError(err)
 	}
-	// get size from request form
-	var objectSize int64
-	if params.HTTPRequest.MultipartForm == nil {
-		return prepareError(errors.New("request MultipartForm is nil"))
-	}
-	if file, ok := params.HTTPRequest.MultipartForm.File["upfile"]; ok {
-		if len(file) > 0 {
-			objectSize = file[0].Size
-		} else {
-			return prepareError(errors.New("file not present in request"))
-		}
-	} else {
-		return prepareError(errors.New("`upfile` should be on MultipartForm.File map"))
-	}
 	// create a minioClient interface implementation
 	// defining the client to be used
 	minioClient := minioClient{client: mClient}
-	if err := uploadObject(ctx, minioClient, params.BucketName, params.Prefix, objectSize, params.Upfile); err != nil {
-		return prepareError(err)
+	if err := uploadFiles(ctx, minioClient, params); err != nil {
+		prepareError(err, errorGeneric)
 	}
 	return nil
 }
 
+// uploadFiles gets files from http.Request form and uploads them to MinIO
+func uploadFiles(ctx context.Context, client MinioClient, params user_api.PostBucketsBucketNameObjectsUploadParams) error {
+	var prefix string
+	if params.Prefix != nil {
+		prefix = *params.Prefix
+	}
+
+	// get object files from request
+	objFiles, err := getFormFiles(params.HTTPRequest)
+	if err != nil {
+		return err
+	}
+
+	// upload files one by one
+	for _, obj := range objFiles {
+		objectPrefix := fmt.Sprintf("%s%s", prefix, obj.name)
+		if err := uploadObject(ctx, client, params.BucketName, objectPrefix, obj.size, obj.file); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type objectFile struct {
+	name string
+	size int64
+	file *runtime.File
+}
+
+// getFormFiles parses the request body and gets all the files from the Request
+// it includes name, size and file content
+func getFormFiles(r *http.Request) (files []*objectFile, err error) {
+	if r == nil {
+		return nil, errors.New("http.Request is nil")
+	}
+	// parse a request body as multipart/form-data.
+	// 32 << 20 is default max memory
+	if err := r.ParseMultipartForm(32 << 20); err != nil {
+		if err != http.ErrNotMultipart {
+			return nil, err
+		} else if err := r.ParseForm(); err != nil {
+			return nil, err
+		}
+	}
+
+	if r.MultipartForm != nil && r.MultipartForm.File != nil {
+		for fileName, file := range r.MultipartForm.File {
+			if fhs := file; len(fhs) > 0 {
+				f, err := fhs[0].Open()
+				if err != nil {
+					return nil, err
+				}
+				of := &objectFile{
+					name: fileName,
+					size: fhs[0].Size,
+					file: &runtime.File{
+						Data:   f,
+						Header: fhs[0],
+					},
+				}
+				files = append(files, of)
+			} else {
+				return nil, errors.New("file not present in request")
+			}
+		}
+	} else {
+		return nil, errors.New("request MultipartForm or MultipartForm.File is nil")
+	}
+	return files, nil
+}
+
 func uploadObject(ctx context.Context, client MinioClient, bucketName, prefix string, objectSize int64, object io.ReadCloser) error {
 	_, err := client.putObject(ctx, bucketName, prefix, object, objectSize, minio.PutObjectOptions{ContentType: "application/octet-stream"})
 	if err != nil {

swagger.yml

@@ -356,17 +356,12 @@ paths:
       consumes:
         - multipart/form-data
       parameters:
-        - in: formData
-          name: upfile
-          type: file
-          required: true
         - name: bucket_name
           in: path
           required: true
           type: string
         - name: prefix
           in: query
-          required: true
           type: string
       responses:
         200: