mirrors/object-browser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix download of large files in Console (#2773)

Browse Source
This commit is contained in:
Javier Adriel
2023-05-15 12:31:43 -06:00
committed by GitHub
parent 22ec87d00e
commit d93537261e
9 changed files with 183 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
portal-ui/package.json
View File

@@ -13,6 +13,7 @@
"@mui/styles": "^5.12.0",
"@mui/x-date-pickers": "^5.0.20",
"@reduxjs/toolkit": "^1.9.5",
"@types/streamsaver": "^2.0.1",
"@uiw/react-textarea-code-editor": "^2.1.1",
"kbar": "^0.1.0-beta.39",
"local-storage-fallback": "^4.1.1",
@@ -31,6 +32,7 @@
"react-window": "^1.8.9",
"react-window-infinite-loader": "^1.0.9",
"recharts": "^2.4.3",
"streamsaver": "^2.0.6",
"styled-components": "^5.3.10",
"superagent": "^8.0.8",
"tinycolor2": "^1.6.0",

2
portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx
View File

@@ -668,9 +668,7 @@ const ListObjects = () => {
errorMessage: "",
})
);
storeFormDataWithID(ID, formData);
storeCallForObjectWithID(ID, xhr);
}
});
};

70
portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx
View File

@@ -33,15 +33,10 @@ import {
textStyleUtils,
} from "../../../../Common/FormComponents/common/styleLibrary";
import { IFileInfo } from "./types";
import { download } from "../utils";
import api from "../../../../../../common/api";
import { ErrorResponseHandler } from "../../../../../../common/types";
import {
decodeURLString,
encodeURLString,
niceBytesInt,
} from "../../../../../../common/utils";
import { decodeURLString, niceBytesInt } from "../../../../../../common/utils";
import RestoreFileVersion from "./RestoreFileVersion";
import { AppState, useAppDispatch } from "../../../../../../store";
@@ -64,21 +59,13 @@ import {
setErrorSnackMessage,
} from "../../../../../../systemSlice";
import {
makeid,
storeCallForObjectWithID,
} from "../../../../ObjectBrowser/transferManager";
import {
cancelObjectInList,
completeObject,
failObject,
setLoadingObjectInfo,
setLoadingVersions,
setNewObject,
setSelectedVersion,
updateProgress,
} from "../../../../ObjectBrowser/objectBrowserSlice";
import { List, ListRowProps } from "react-virtualized";
import TooltipWrapper from "../../../../Common/TooltipWrapper/TooltipWrapper";
import { downloadObject } from "../../../../ObjectBrowser/utils";
const styles = (theme: Theme) =>
createStyles({
@@ -237,57 +224,6 @@ const VersionsNavigator = ({
setPreviewOpen(false);
};
const downloadObject = (object: IFileInfo) => {
const identityDownload = encodeURLString(
`${bucketName}-${object.name}-${new Date().getTime()}-${Math.random()}`
);
const ID = makeid(8);
const downloadCall = download(
bucketName,
internalPaths,
object.version_id,
parseInt(object.size || "0"),
null,
ID,
(progress) => {
dispatch(
updateProgress({
instanceID: identityDownload,
progress: progress,
})
);
},
() => {
dispatch(completeObject(identityDownload));
},
(msg: string) => {
dispatch(failObject({ instanceID: identityDownload, msg }));
},
() => {
dispatch(cancelObjectInList(identityDownload));
}
);
storeCallForObjectWithID(ID, downloadCall);
dispatch(
setNewObject({
ID,
bucketName,
done: false,
instanceID: identityDownload,
percentage: 0,
prefix: object.name,
type: "download",
waitingForFile: true,
failed: false,
cancelled: false,
errorMessage: "",
})
);
};
const onShareItem = (item: IFileInfo) => {
setObjectToShare(item);
shareObject();
@@ -304,7 +240,7 @@ const VersionsNavigator = ({
};
const onDownloadItem = (item: IFileInfo) => {
downloadObject(item);
downloadObject(dispatch, bucketName, internalPaths, item);
};
const onGlobalClick = (item: IFileInfo) => {

216
portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts
View File

@@ -18,6 +18,7 @@ import { BucketObjectItem } from "./ListObjects/types";
import { IAllowResources } from "../../../types";
import { encodeURLString } from "../../../../../common/utils";
import { removeTrace } from "../../../ObjectBrowser/transferManager";
import streamSaver from "streamsaver";
import store from "../../../../../store";
export const download = (
@@ -30,7 +31,8 @@ export const download = (
progressCallback: (progress: number) => void,
completeCallback: () => void,
errorCallback: (msg: string) => void,
abortCallback: () => void
abortCallback: () => void,
toastCallback: () => void
) => {
const anchor = document.createElement("a");
document.body.appendChild(anchor);
@@ -48,76 +50,154 @@ export const download = (
if (versionID) {
path = path.concat(`&version_id=${versionID}`);
}
var req = new XMLHttpRequest();
req.open("GET", path, true);
if (anonymousMode) {
req.setRequestHeader("X-Anonymous", "1");
}
req.addEventListener(
"progress",
function (evt) {
let percentComplete = Math.round((evt.loaded / fileSize) * 100);
if (progressCallback) {
progressCallback(percentComplete);
}
},
false
return new DownloadHelper(
path,
id,
anonymousMode,
fileSize,
progressCallback,
completeCallback,
errorCallback,
abortCallback,
toastCallback
);
req.responseType = "blob";
req.onreadystatechange = () => {
if (req.readyState === 4) {
if (req.status === 200) {
const rspHeader = req.getResponseHeader("Content-Disposition");
let filename = "download";
if (rspHeader) {
let rspHeaderDecoded = decodeURIComponent(rspHeader);
filename = rspHeaderDecoded.split('"')[1];
}
if (completeCallback) {
completeCallback();
}
removeTrace(id);
var link = document.createElement("a");
link.href = window.URL.createObjectURL(req.response);
link.download = filename;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
} else {
if (req.getResponseHeader("Content-Type") === "application/json") {
const rspBody: { detailedMessage?: string } = JSON.parse(
req.response
);
if (rspBody.detailedMessage) {
errorCallback(rspBody.detailedMessage);
return;
}
}
errorCallback(`Unexpected response status code (${req.status}).`);
}
}
};
req.onerror = () => {
if (errorCallback) {
errorCallback("A network error occurred.");
}
};
req.onabort = () => {
if (abortCallback) {
abortCallback();
}
};
return req;
};
class DownloadHelper {
aborter: AbortController;
path: string;
id: string;
filename: string = "";
anonymousMode: boolean;
fileSize: number = 0;
writer: any = null;
progressCallback: (progress: number) => void;
completeCallback: () => void;
errorCallback: (msg: string) => void;
abortCallback: () => void;
toastCallback: () => void;
constructor(
path: string,
id: string,
anonymousMode: boolean,
fileSize: number,
progressCallback: (progress: number) => void,
completeCallback: () => void,
errorCallback: (msg: string) => void,
abortCallback: () => void,
toastCallback: () => void
) {
this.aborter = new AbortController();
this.path = path;
this.id = id;
this.anonymousMode = anonymousMode;
this.fileSize = fileSize;
this.progressCallback = progressCallback;
this.completeCallback = completeCallback;
this.errorCallback = errorCallback;
this.abortCallback = abortCallback;
this.toastCallback = toastCallback;
}
abort(): void {
this.aborter.abort();
this.abortCallback();
if (this.writer) {
this.writer.abort();
}
}
send(): void {
let isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
if (isSafari) {
this.toastCallback();
this.downloadSafari();
} else {
this.download({
url: this.path,
chunkSize: 1024 * 1024 * 1024 * 1.5,
});
}
}
async getRangeContent(url: string, start: number, end: number) {
const info = this.getRequestInfo(start, end);
const response = await fetch(url, info);
if (response.ok && response.body) {
if (!this.filename) {
this.filename = this.getFilename(response);
}
if (!this.writer) {
this.writer = streamSaver.createWriteStream(this.filename).getWriter();
}
const reader = response.body.getReader();
let done, value;
while (!done) {
({ value, done } = await reader.read());
if (done) {
break;
}
await this.writer.write(value);
}
} else {
throw new Error(`Unexpected response status code (${response.status}).`);
}
}
getRequestInfo(start: number, end: number) {
const info: RequestInit = {
signal: this.aborter.signal,
headers: { range: `bytes=${start}-${end}` },
};
if (this.anonymousMode) {
info.headers = { ...info.headers, "X-Anonymous": "1" };
}
return info;
}
getFilename(response: Response) {
const rspHeader = response.headers.get("Content-Disposition");
if (rspHeader) {
let rspHeaderDecoded = decodeURIComponent(rspHeader);
return rspHeaderDecoded.split('"')[1];
}
return "download";
}
async download({ url, chunkSize }: any) {
const numberOfChunks = Math.ceil(this.fileSize / chunkSize);
this.progressCallback(0);
try {
for (let i = 0; i < numberOfChunks; i++) {
let start = i * chunkSize;
let end =
i + 1 === numberOfChunks
? this.fileSize - 1
: (i + 1) * chunkSize - 1;
await this.getRangeContent(url, start, end);
let percentComplete = Math.round(((i + 1) / numberOfChunks) * 100);
this.progressCallback(percentComplete);
}
this.writer.close();
this.completeCallback();
removeTrace(this.id);
} catch (e: any) {
this.errorCallback(e.message);
}
}
downloadSafari() {
const link = document.createElement("a");
link.href = this.path;
document.body.appendChild(link);
link.click();
document.body.removeChild(link);
this.completeCallback();
removeTrace(this.id);
}
}
// Review file extension by name & returns the type of preview browser that can be used
export const extensionPreview = (
fileName: string

63
portal-ui/src/screens/Console/ObjectBrowser/RenameLongFilename.tsx
View File

@@ -27,20 +27,11 @@ import {
spacingUtils,
} from "../Common/FormComponents/common/styleLibrary";
import { IFileInfo } from "../Buckets/ListBuckets/Objects/ObjectDetails/types";
import { encodeURLString } from "../../../common/utils";
import { download } from "../Buckets/ListBuckets/Objects/utils";
import {
cancelObjectInList,
completeObject,
failObject,
setNewObject,
updateProgress,
} from "./objectBrowserSlice";
import { makeid, storeCallForObjectWithID } from "./transferManager";
import { useAppDispatch } from "../../../store";
import ModalWrapper from "../Common/ModalWrapper/ModalWrapper";
import InputBoxWrapper from "../Common/FormComponents/InputBoxWrapper/InputBoxWrapper";
import FormSwitchWrapper from "../Common/FormComponents/FormSwitchWrapper/FormSwitchWrapper";
import { downloadObject } from "./utils";
interface IRenameLongFilename {
open: boolean;
@@ -76,57 +67,7 @@ const RenameLongFileName = ({
const doDownload = (e: React.FormEvent<HTMLFormElement>) => {
e.preventDefault();
const identityDownload = encodeURLString(
`${bucketName}-${
actualInfo.name
}-${new Date().getTime()}-${Math.random()}`
);
const ID = makeid(8);
const downloadCall = download(
bucketName,
internalPaths,
actualInfo.version_id,
parseInt(actualInfo.size || "0"),
newFileName,
ID,
(progress) => {
dispatch(
updateProgress({
instanceID: identityDownload,
progress: progress,
})
);
},
() => {
dispatch(completeObject(identityDownload));
},
(msg: string) => {
dispatch(failObject({ instanceID: identityDownload, msg }));
},
() => {
dispatch(cancelObjectInList(identityDownload));
}
);
storeCallForObjectWithID(ID, downloadCall);
dispatch(
setNewObject({
ID,
bucketName,
done: false,
instanceID: identityDownload,
percentage: 0,
prefix: newFileName,
type: "download",
waitingForFile: true,
failed: false,
cancelled: false,
errorMessage: "",
})
);
downloadObject(dispatch, bucketName, internalPaths, actualInfo);
closeModal();
};

8
portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts
View File

@@ -32,6 +32,7 @@ import {
setShareFileModalOpen,
updateProgress,
} from "./objectBrowserSlice";
import { setSnackBarMessage } from "../../../systemSlice";
export const downloadSelected = createAsyncThunk(
"objectBrowser/downloadSelected",
@@ -68,6 +69,13 @@ export const downloadSelected = createAsyncThunk(
},
() => {
dispatch(cancelObjectInList(identityDownload));
},
() => {
dispatch(
setSnackBarMessage(
"File download will be handled directly by the browser."
)
);
}
);
storeCallForObjectWithID(ID, downloadCall);

4
portal-ui/src/screens/Console/ObjectBrowser/transferManager.ts
View File

@@ -17,11 +17,11 @@
let objectCalls: { [key: string]: XMLHttpRequest } = {};
let formDataElements: { [key: string]: FormData } = {};
export const storeCallForObjectWithID = (id: string, call: XMLHttpRequest) => {
export const storeCallForObjectWithID = (id: string, call: any) => {
objectCalls[id] = call;
};
export const callForObjectID = (id: string): XMLHttpRequest => {
export const callForObjectID = (id: string): any => {
return objectCalls[id];
};

8
portal-ui/src/screens/Console/ObjectBrowser/utils.ts
View File

@@ -27,6 +27,7 @@ import {
updateProgress,
} from "./objectBrowserSlice";
import { AppDispatch } from "../../../store";
import { setSnackBarMessage } from "../../../systemSlice";
export const downloadObject = (
dispatch: AppDispatch,
@@ -68,6 +69,13 @@ export const downloadObject = (
},
() => {
dispatch(cancelObjectInList(identityDownload));
},
() => {
dispatch(
setSnackBarMessage(
"File download will be handled directly by the browser."
)
);
}
);

10
portal-ui/yarn.lock
View File

@@ -2621,6 +2621,11 @@
resolved "https://registry.yarnpkg.com/@types/stack-utils/-/stack-utils-2.0.1.tgz#20f18294f797f2209b5f65c8e3b5c8e8261d127c"
integrity sha512-Hl219/BT5fLAaz6NDkSuhzasy49dwQS/DSdu4MdggFB8zcXv7vflBI3xp7FEmkmdDkBUI2bPUNeMttp2knYdxw==
"@types/streamsaver@^2.0.1":
version "2.0.1"
resolved "https://registry.yarnpkg.com/@types/streamsaver/-/streamsaver-2.0.1.tgz#fa5e5b891d1b282be3078c232a30ee004b8e0be0"
integrity sha512-I49NtT8w6syBI3Zg3ixCyygTHoTVMY0z2TMRcTgccdIsVd2MwlKk7ITLHLsJtgchUHcOd7QEARG9h0ifcA6l2Q==
"@types/styled-components@^5.1.25":
version "5.1.26"
resolved "https://registry.yarnpkg.com/@types/styled-components/-/styled-components-5.1.26.tgz#5627e6812ee96d755028a98dae61d28e57c233af"
@@ -11092,6 +11097,11 @@ stop-iteration-iterator@^1.0.0:
dependencies:
internal-slot "^1.0.4"
streamsaver@^2.0.6:
version "2.0.6"
resolved "https://registry.yarnpkg.com/streamsaver/-/streamsaver-2.0.6.tgz#869d2347dd70191e0ac888d52296956a8cba2ed9"
integrity sha512-LK4e7TfCV8HzuM0PKXuVUfKyCB1FtT9L0EGxsFk5Up8njj0bXK8pJM9+Wq2Nya7/jslmCQwRK39LFm55h7NBTw==
strict-uri-encode@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/strict-uri-encode/-/strict-uri-encode-2.0.0.tgz#b9c7330c7042862f6b142dc274bbcc5866ce3546"