From d93537261ee39c04829f027fd1267f657f3e9a22 Mon Sep 17 00:00:00 2001 From: Javier Adriel Date: Mon, 15 May 2023 12:31:43 -0600 Subject: [PATCH] Fix download of large files in Console (#2773) --- portal-ui/package.json | 2 + .../Objects/ListObjects/ListObjects.tsx | 2 - .../ObjectDetails/VersionsNavigator.tsx | 70 +----- .../Buckets/ListBuckets/Objects/utils.ts | 216 ++++++++++++------ .../ObjectBrowser/RenameLongFilename.tsx | 63 +---- .../ObjectBrowser/objectBrowserThunks.ts | 8 + .../Console/ObjectBrowser/transferManager.ts | 4 +- .../screens/Console/ObjectBrowser/utils.ts | 8 + portal-ui/yarn.lock | 10 + 9 files changed, 183 insertions(+), 200 deletions(-) diff --git a/portal-ui/package.json b/portal-ui/package.json index afe750c56..08669ae6c 100644 --- a/portal-ui/package.json +++ b/portal-ui/package.json @@ -13,6 +13,7 @@ "@mui/styles": "^5.12.0", "@mui/x-date-pickers": "^5.0.20", "@reduxjs/toolkit": "^1.9.5", + "@types/streamsaver": "^2.0.1", "@uiw/react-textarea-code-editor": "^2.1.1", "kbar": "^0.1.0-beta.39", "local-storage-fallback": "^4.1.1", @@ -31,6 +32,7 @@ "react-window": "^1.8.9", "react-window-infinite-loader": "^1.0.9", "recharts": "^2.4.3", + "streamsaver": "^2.0.6", "styled-components": "^5.3.10", "superagent": "^8.0.8", "tinycolor2": "^1.6.0", diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx index dae29bed8..98364d043 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx @@ -668,9 +668,7 @@ const ListObjects = () => { errorMessage: "", }) ); - storeFormDataWithID(ID, formData); - storeCallForObjectWithID(ID, xhr); } }); }; diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx index 5abb1bfa8..744ad103c 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx @@ -33,15 +33,10 @@ import { textStyleUtils, } from "../../../../Common/FormComponents/common/styleLibrary"; import { IFileInfo } from "./types"; -import { download } from "../utils"; import api from "../../../../../../common/api"; import { ErrorResponseHandler } from "../../../../../../common/types"; -import { - decodeURLString, - encodeURLString, - niceBytesInt, -} from "../../../../../../common/utils"; +import { decodeURLString, niceBytesInt } from "../../../../../../common/utils"; import RestoreFileVersion from "./RestoreFileVersion"; import { AppState, useAppDispatch } from "../../../../../../store"; @@ -64,21 +59,13 @@ import { setErrorSnackMessage, } from "../../../../../../systemSlice"; import { - makeid, - storeCallForObjectWithID, -} from "../../../../ObjectBrowser/transferManager"; -import { - cancelObjectInList, - completeObject, - failObject, setLoadingObjectInfo, setLoadingVersions, - setNewObject, setSelectedVersion, - updateProgress, } from "../../../../ObjectBrowser/objectBrowserSlice"; import { List, ListRowProps } from "react-virtualized"; import TooltipWrapper from "../../../../Common/TooltipWrapper/TooltipWrapper"; +import { downloadObject } from "../../../../ObjectBrowser/utils"; const styles = (theme: Theme) => createStyles({ @@ -237,57 +224,6 @@ const VersionsNavigator = ({ setPreviewOpen(false); }; - const downloadObject = (object: IFileInfo) => { - const identityDownload = encodeURLString( - `${bucketName}-${object.name}-${new Date().getTime()}-${Math.random()}` - ); - - const ID = makeid(8); - - const downloadCall = download( - bucketName, - internalPaths, - object.version_id, - parseInt(object.size || "0"), - null, - ID, - (progress) => { - dispatch( - updateProgress({ - instanceID: identityDownload, - progress: progress, - }) - ); - }, - () => { - dispatch(completeObject(identityDownload)); - }, - (msg: string) => { - dispatch(failObject({ instanceID: identityDownload, msg })); - }, - () => { - dispatch(cancelObjectInList(identityDownload)); - } - ); - - storeCallForObjectWithID(ID, downloadCall); - dispatch( - setNewObject({ - ID, - bucketName, - done: false, - instanceID: identityDownload, - percentage: 0, - prefix: object.name, - type: "download", - waitingForFile: true, - failed: false, - cancelled: false, - errorMessage: "", - }) - ); - }; - const onShareItem = (item: IFileInfo) => { setObjectToShare(item); shareObject(); @@ -304,7 +240,7 @@ const VersionsNavigator = ({ }; const onDownloadItem = (item: IFileInfo) => { - downloadObject(item); + downloadObject(dispatch, bucketName, internalPaths, item); }; const onGlobalClick = (item: IFileInfo) => { diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts index 22ea370c0..e4bf97131 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts @@ -18,6 +18,7 @@ import { BucketObjectItem } from "./ListObjects/types"; import { IAllowResources } from "../../../types"; import { encodeURLString } from "../../../../../common/utils"; import { removeTrace } from "../../../ObjectBrowser/transferManager"; +import streamSaver from "streamsaver"; import store from "../../../../../store"; export const download = ( @@ -30,7 +31,8 @@ export const download = ( progressCallback: (progress: number) => void, completeCallback: () => void, errorCallback: (msg: string) => void, - abortCallback: () => void + abortCallback: () => void, + toastCallback: () => void ) => { const anchor = document.createElement("a"); document.body.appendChild(anchor); @@ -48,76 +50,154 @@ export const download = ( if (versionID) { path = path.concat(`&version_id=${versionID}`); } - - var req = new XMLHttpRequest(); - req.open("GET", path, true); - if (anonymousMode) { - req.setRequestHeader("X-Anonymous", "1"); - } - req.addEventListener( - "progress", - function (evt) { - let percentComplete = Math.round((evt.loaded / fileSize) * 100); - - if (progressCallback) { - progressCallback(percentComplete); - } - }, - false + return new DownloadHelper( + path, + id, + anonymousMode, + fileSize, + progressCallback, + completeCallback, + errorCallback, + abortCallback, + toastCallback ); - - req.responseType = "blob"; - req.onreadystatechange = () => { - if (req.readyState === 4) { - if (req.status === 200) { - const rspHeader = req.getResponseHeader("Content-Disposition"); - - let filename = "download"; - if (rspHeader) { - let rspHeaderDecoded = decodeURIComponent(rspHeader); - filename = rspHeaderDecoded.split('"')[1]; - } - - if (completeCallback) { - completeCallback(); - } - - removeTrace(id); - - var link = document.createElement("a"); - link.href = window.URL.createObjectURL(req.response); - link.download = filename; - document.body.appendChild(link); - link.click(); - document.body.removeChild(link); - } else { - if (req.getResponseHeader("Content-Type") === "application/json") { - const rspBody: { detailedMessage?: string } = JSON.parse( - req.response - ); - if (rspBody.detailedMessage) { - errorCallback(rspBody.detailedMessage); - return; - } - } - errorCallback(`Unexpected response status code (${req.status}).`); - } - } - }; - req.onerror = () => { - if (errorCallback) { - errorCallback("A network error occurred."); - } - }; - req.onabort = () => { - if (abortCallback) { - abortCallback(); - } - }; - - return req; }; +class DownloadHelper { + aborter: AbortController; + path: string; + id: string; + filename: string = ""; + anonymousMode: boolean; + fileSize: number = 0; + writer: any = null; + progressCallback: (progress: number) => void; + completeCallback: () => void; + errorCallback: (msg: string) => void; + abortCallback: () => void; + toastCallback: () => void; + + constructor( + path: string, + id: string, + anonymousMode: boolean, + fileSize: number, + progressCallback: (progress: number) => void, + completeCallback: () => void, + errorCallback: (msg: string) => void, + abortCallback: () => void, + toastCallback: () => void + ) { + this.aborter = new AbortController(); + this.path = path; + this.id = id; + this.anonymousMode = anonymousMode; + this.fileSize = fileSize; + this.progressCallback = progressCallback; + this.completeCallback = completeCallback; + this.errorCallback = errorCallback; + this.abortCallback = abortCallback; + this.toastCallback = toastCallback; + } + + abort(): void { + this.aborter.abort(); + this.abortCallback(); + if (this.writer) { + this.writer.abort(); + } + } + + send(): void { + let isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + if (isSafari) { + this.toastCallback(); + this.downloadSafari(); + } else { + this.download({ + url: this.path, + chunkSize: 1024 * 1024 * 1024 * 1.5, + }); + } + } + + async getRangeContent(url: string, start: number, end: number) { + const info = this.getRequestInfo(start, end); + const response = await fetch(url, info); + if (response.ok && response.body) { + if (!this.filename) { + this.filename = this.getFilename(response); + } + if (!this.writer) { + this.writer = streamSaver.createWriteStream(this.filename).getWriter(); + } + const reader = response.body.getReader(); + let done, value; + while (!done) { + ({ value, done } = await reader.read()); + if (done) { + break; + } + await this.writer.write(value); + } + } else { + throw new Error(`Unexpected response status code (${response.status}).`); + } + } + + getRequestInfo(start: number, end: number) { + const info: RequestInit = { + signal: this.aborter.signal, + headers: { range: `bytes=${start}-${end}` }, + }; + if (this.anonymousMode) { + info.headers = { ...info.headers, "X-Anonymous": "1" }; + } + return info; + } + + getFilename(response: Response) { + const rspHeader = response.headers.get("Content-Disposition"); + if (rspHeader) { + let rspHeaderDecoded = decodeURIComponent(rspHeader); + return rspHeaderDecoded.split('"')[1]; + } + return "download"; + } + + async download({ url, chunkSize }: any) { + const numberOfChunks = Math.ceil(this.fileSize / chunkSize); + this.progressCallback(0); + try { + for (let i = 0; i < numberOfChunks; i++) { + let start = i * chunkSize; + let end = + i + 1 === numberOfChunks + ? this.fileSize - 1 + : (i + 1) * chunkSize - 1; + await this.getRangeContent(url, start, end); + let percentComplete = Math.round(((i + 1) / numberOfChunks) * 100); + this.progressCallback(percentComplete); + } + this.writer.close(); + this.completeCallback(); + removeTrace(this.id); + } catch (e: any) { + this.errorCallback(e.message); + } + } + + downloadSafari() { + const link = document.createElement("a"); + link.href = this.path; + document.body.appendChild(link); + link.click(); + document.body.removeChild(link); + this.completeCallback(); + removeTrace(this.id); + } +} + // Review file extension by name & returns the type of preview browser that can be used export const extensionPreview = ( fileName: string diff --git a/portal-ui/src/screens/Console/ObjectBrowser/RenameLongFilename.tsx b/portal-ui/src/screens/Console/ObjectBrowser/RenameLongFilename.tsx index 268ebeceb..9ff1a5390 100644 --- a/portal-ui/src/screens/Console/ObjectBrowser/RenameLongFilename.tsx +++ b/portal-ui/src/screens/Console/ObjectBrowser/RenameLongFilename.tsx @@ -27,20 +27,11 @@ import { spacingUtils, } from "../Common/FormComponents/common/styleLibrary"; import { IFileInfo } from "../Buckets/ListBuckets/Objects/ObjectDetails/types"; -import { encodeURLString } from "../../../common/utils"; -import { download } from "../Buckets/ListBuckets/Objects/utils"; -import { - cancelObjectInList, - completeObject, - failObject, - setNewObject, - updateProgress, -} from "./objectBrowserSlice"; -import { makeid, storeCallForObjectWithID } from "./transferManager"; import { useAppDispatch } from "../../../store"; import ModalWrapper from "../Common/ModalWrapper/ModalWrapper"; import InputBoxWrapper from "../Common/FormComponents/InputBoxWrapper/InputBoxWrapper"; import FormSwitchWrapper from "../Common/FormComponents/FormSwitchWrapper/FormSwitchWrapper"; +import { downloadObject } from "./utils"; interface IRenameLongFilename { open: boolean; @@ -76,57 +67,7 @@ const RenameLongFileName = ({ const doDownload = (e: React.FormEvent) => { e.preventDefault(); - - const identityDownload = encodeURLString( - `${bucketName}-${ - actualInfo.name - }-${new Date().getTime()}-${Math.random()}` - ); - - const ID = makeid(8); - - const downloadCall = download( - bucketName, - internalPaths, - actualInfo.version_id, - parseInt(actualInfo.size || "0"), - newFileName, - ID, - (progress) => { - dispatch( - updateProgress({ - instanceID: identityDownload, - progress: progress, - }) - ); - }, - () => { - dispatch(completeObject(identityDownload)); - }, - (msg: string) => { - dispatch(failObject({ instanceID: identityDownload, msg })); - }, - () => { - dispatch(cancelObjectInList(identityDownload)); - } - ); - - storeCallForObjectWithID(ID, downloadCall); - dispatch( - setNewObject({ - ID, - bucketName, - done: false, - instanceID: identityDownload, - percentage: 0, - prefix: newFileName, - type: "download", - waitingForFile: true, - failed: false, - cancelled: false, - errorMessage: "", - }) - ); + downloadObject(dispatch, bucketName, internalPaths, actualInfo); closeModal(); }; diff --git a/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts b/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts index ea7458cde..312ddd3a1 100644 --- a/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts +++ b/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts @@ -32,6 +32,7 @@ import { setShareFileModalOpen, updateProgress, } from "./objectBrowserSlice"; +import { setSnackBarMessage } from "../../../systemSlice"; export const downloadSelected = createAsyncThunk( "objectBrowser/downloadSelected", @@ -68,6 +69,13 @@ export const downloadSelected = createAsyncThunk( }, () => { dispatch(cancelObjectInList(identityDownload)); + }, + () => { + dispatch( + setSnackBarMessage( + "File download will be handled directly by the browser." + ) + ); } ); storeCallForObjectWithID(ID, downloadCall); diff --git a/portal-ui/src/screens/Console/ObjectBrowser/transferManager.ts b/portal-ui/src/screens/Console/ObjectBrowser/transferManager.ts index b4bc081eb..e3af37d9a 100644 --- a/portal-ui/src/screens/Console/ObjectBrowser/transferManager.ts +++ b/portal-ui/src/screens/Console/ObjectBrowser/transferManager.ts @@ -17,11 +17,11 @@ let objectCalls: { [key: string]: XMLHttpRequest } = {}; let formDataElements: { [key: string]: FormData } = {}; -export const storeCallForObjectWithID = (id: string, call: XMLHttpRequest) => { +export const storeCallForObjectWithID = (id: string, call: any) => { objectCalls[id] = call; }; -export const callForObjectID = (id: string): XMLHttpRequest => { +export const callForObjectID = (id: string): any => { return objectCalls[id]; }; diff --git a/portal-ui/src/screens/Console/ObjectBrowser/utils.ts b/portal-ui/src/screens/Console/ObjectBrowser/utils.ts index f6e48d8c7..b198d133f 100644 --- a/portal-ui/src/screens/Console/ObjectBrowser/utils.ts +++ b/portal-ui/src/screens/Console/ObjectBrowser/utils.ts @@ -27,6 +27,7 @@ import { updateProgress, } from "./objectBrowserSlice"; import { AppDispatch } from "../../../store"; +import { setSnackBarMessage } from "../../../systemSlice"; export const downloadObject = ( dispatch: AppDispatch, @@ -68,6 +69,13 @@ export const downloadObject = ( }, () => { dispatch(cancelObjectInList(identityDownload)); + }, + () => { + dispatch( + setSnackBarMessage( + "File download will be handled directly by the browser." + ) + ); } ); diff --git a/portal-ui/yarn.lock b/portal-ui/yarn.lock index f1a1cb6a4..acc0d2910 100644 --- a/portal-ui/yarn.lock +++ b/portal-ui/yarn.lock @@ -2621,6 +2621,11 @@ resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c" integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw== +"@types/streamsaver@^2.0.1": + version "2.0.1" + resolved "https://registry.yarnpkg.com/@types/streamsaver/-/streamsaver-2.0.1.tgz#fa5e5b891d1b282be3078c232a30ee004b8e0be0" + integrity sha512-I49NtT8w6syBI3Zg3ixCyygTHoTVMY0z2TMRcTgccdIsVd2MwlKk7ITLHLsJtgchUHcOd7QEARG9h0ifcA6l2Q== + "@types/styled-components@^5.1.25": version "5.1.26" resolved "https://registry.yarnpkg.com/@types/styled-components/-/styled-components-5.1.26.tgz#5627e6812ee96d755028a98dae61d28e57c233af" @@ -11092,6 +11097,11 @@ stop-iteration-iterator@^1.0.0: dependencies: internal-slot "^1.0.4" +streamsaver@^2.0.6: + version "2.0.6" + resolved "https://registry.yarnpkg.com/streamsaver/-/streamsaver-2.0.6.tgz#869d2347dd70191e0ac888d52296956a8cba2ed9" + integrity sha512-LK4e7TfCV8HzuM0PKXuVUfKyCB1FtT9L0EGxsFk5Up8njj0bXK8pJM9+Wq2Nya7/jslmCQwRK39LFm55h7NBTw== + strict-uri-encode@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546"