From d95d59e45413d525f121a4248899a1559c71de0c Mon Sep 17 00:00:00 2001 From: Alex <33497058+bexsoft@users.noreply.github.com> Date: Tue, 6 Dec 2022 13:23:07 -0600 Subject: [PATCH] Changed Object browser logic to work with websockets (#2419) fixes https://github.com/minio/console/issues/943 ## What does this do? This allows us to start streaming results to the list instead of waiting to retrieve all the objects list before sending it to the client Included a couple of fixes: - Removed metadata for deleted items - Fixed multiple metadata requests - Fixed height grow for parent wrapper - Fixed object reload after restore version Signed-off-by: Benjamin Perez --- models/rewind_item.go | 3 + .../Buckets/BucketDetails/BrowserHandler.tsx | 397 ++++++++- .../Buckets/ListBuckets/ListBuckets.tsx | 2 + .../Objects/ListObjects/CreatePathModal.tsx | 8 +- .../Objects/ListObjects/ListObjects.tsx | 755 +++--------------- .../Objects/ListObjects/ListObjectsTable.tsx | 246 ++++++ .../Objects/ListObjects/ObjectDetailPanel.tsx | 65 +- .../ListBuckets/Objects/ListObjects/types.tsx | 33 +- .../Objects/ObjectDetails/ObjectMetaData.tsx | 57 +- .../ObjectDetails/RestoreFileVersion.tsx | 17 +- .../ObjectDetails/VersionsNavigator.tsx | 13 +- .../Buckets/ListBuckets/Objects/utils.ts | 4 +- .../ObjectBrowser/BrowserBreadcrumbs.tsx | 4 - .../ObjectBrowser/objectBrowserSlice.ts | 91 +++ .../ObjectBrowser/objectBrowserThunks.ts | 157 ++++ .../screens/Console/ObjectBrowser/types.ts | 13 + restapi/admin_objects.go | 102 +++ restapi/admin_objects_test.go | 236 ++++++ restapi/embedded_spec.go | 6 + restapi/user_objects.go | 1 + restapi/ws_handle.go | 284 ++++++- swagger-console.yml | 2 + 22 files changed, 1745 insertions(+), 751 deletions(-) create mode 100644 portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjectsTable.tsx create mode 100644 portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts create mode 100644 restapi/admin_objects.go create mode 100644 restapi/admin_objects_test.go diff --git a/models/rewind_item.go b/models/rewind_item.go index 1b0813d3a..0b089e3b6 100644 --- a/models/rewind_item.go +++ b/models/rewind_item.go @@ -40,6 +40,9 @@ type RewindItem struct { // delete flag DeleteFlag bool `json:"delete_flag,omitempty"` + // is latest + IsLatest bool `json:"is_latest,omitempty"` + // last modified LastModified string `json:"last_modified,omitempty"` diff --git a/portal-ui/src/screens/Console/Buckets/BucketDetails/BrowserHandler.tsx b/portal-ui/src/screens/Console/Buckets/BucketDetails/BrowserHandler.tsx index 5188f365a..fb687580d 100644 --- a/portal-ui/src/screens/Console/Buckets/BucketDetails/BrowserHandler.tsx +++ b/portal-ui/src/screens/Console/Buckets/BucketDetails/BrowserHandler.tsx @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . -import React, { Fragment, useEffect } from "react"; +import React, { Fragment, useCallback, useEffect } from "react"; import { useSelector } from "react-redux"; import { useLocation, useNavigate, useParams } from "react-router-dom"; import { Theme } from "@mui/material/styles"; @@ -37,8 +37,22 @@ import { } from "../../../../common/SecureComponent/permissions"; import BackLink from "../../../../common/BackLink"; import { + newMessage, + resetMessages, + setIsVersioned, + setLoadingLocking, + setLoadingObjectInfo, + setLoadingObjectsList, + setLoadingRecords, + setLoadingVersioning, + setLoadingVersions, + setLockingEnabled, + setObjectDetailsView, + setRecords, setSearchObjects, setSearchVersions, + setSelectedObjectView, + setSimplePathHandler, setVersionsModeEnabled, } from "../../ObjectBrowser/objectBrowserSlice"; import SearchBox from "../../Common/SearchBox"; @@ -47,18 +61,93 @@ import AutoColorIcon from "../../Common/Components/AutoColorIcon"; import TooltipWrapper from "../../Common/TooltipWrapper/TooltipWrapper"; import { Button } from "mds"; import hasPermission from "../../../../common/SecureComponent/accessControl"; +import { IMessageEvent } from "websocket"; +import { wsProtocol } from "../../../../utils/wsUtils"; +import { + WebsocketRequest, + WebsocketResponse, +} from "../ListBuckets/Objects/ListObjects/types"; +import { decodeURLString, encodeURLString } from "../../../../common/utils"; +import { permissionItems } from "../ListBuckets/Objects/utils"; +import { setErrorSnackMessage } from "../../../../systemSlice"; +import api from "../../../../common/api"; +import { BucketObjectLocking, BucketVersioning } from "../types"; +import { ErrorResponseHandler } from "../../../../common/types"; const styles = (theme: Theme) => createStyles({ ...containerForHeader(theme.spacing(4)), }); +let objectsWS: WebSocket; +let currentRequestID: number = 0; +let errorCounter: number = 0; + +const initWSConnection = ( + onMessageCallback: (message: IMessageEvent) => void, + openCallback?: () => void, + notAvailableCallback?: () => void +) => { + const url = new URL(window.location.toString()); + const isDev = process.env.NODE_ENV === "development"; + const port = isDev ? "9090" : url.port; + + // check if we are using base path, if not this always is `/` + const baseLocation = new URL(document.baseURI); + const baseUrl = baseLocation.pathname; + + const wsProt = wsProtocol(url.protocol); + + objectsWS = new WebSocket( + `${wsProt}://${url.hostname}:${port}${baseUrl}ws/objectManager` + ); + + objectsWS.onopen = () => { + if (openCallback) { + openCallback(); + } + errorCounter = 0; + }; + + const reconnectFn = () => { + if (errorCounter <= 5) { + initWSConnection(onMessageCallback, openCallback); + errorCounter += 1; + } else { + console.error("Websocket not available."); + if (notAvailableCallback) { + notAvailableCallback(); + } + } + }; + + objectsWS.onclose = () => { + console.warn("Websocket Disconnected. Attempting Reconnection..."); + + // We reconnect after 3 seconds + setTimeout(reconnectFn, 3000); + }; + + objectsWS.onerror = () => { + console.error("Error in websocket connection. Attempting reconnection..."); + + // We reconnect after 3 seconds + setTimeout(reconnectFn, 3000); + }; +}; + +initWSConnection(() => {}); + const BrowserHandler = () => { const dispatch = useAppDispatch(); const navigate = useNavigate(); const params = useParams(); const location = useLocation(); + const loadingVersioning = useSelector( + (state: AppState) => state.objectBrowser.loadingVersioning + ); + const versionsMode = useSelector( (state: AppState) => state.objectBrowser.versionsMode ); @@ -71,6 +160,36 @@ const BrowserHandler = () => { const searchVersions = useSelector( (state: AppState) => state.objectBrowser.searchVersions ); + const rewindEnabled = useSelector( + (state: AppState) => state.objectBrowser.rewind.rewindEnabled + ); + const rewindDate = useSelector( + (state: AppState) => state.objectBrowser.rewind.dateToRewind + ); + const showDeleted = useSelector( + (state: AppState) => state.objectBrowser.showDeleted + ); + const allowResources = useSelector( + (state: AppState) => state.console.session.allowResources + ); + const loadingObjects = useSelector( + (state: AppState) => state.objectBrowser.loadingObjects + ); + const loadingLocking = useSelector( + (state: AppState) => state.objectBrowser.loadingLocking + ); + const bucketToRewind = useSelector( + (state: AppState) => state.objectBrowser.rewind.bucketToRewind + ); + const loadRecords = useSelector( + (state: AppState) => state.objectBrowser.loadRecords + ); + const detailsOpen = useSelector( + (state: AppState) => state.objectBrowser.objectDetailsOpen + ); + const selectedInternalPaths = useSelector( + (state: AppState) => state.objectBrowser.selectedInternalPaths + ); const features = useSelector(selFeatures); @@ -81,10 +200,286 @@ const BrowserHandler = () => { const obOnly = !!features?.includes("object-browser-only"); + /*WS Request Handlers*/ + objectsWS.onmessage = useCallback( + (message: IMessageEvent) => { + // reset start status + dispatch(setLoadingObjectsList(false)); + + const response: WebsocketResponse = JSON.parse(message.data.toString()); + if (currentRequestID === response.request_id) { + // If response is not from current request, we can omit + if (response.request_id !== currentRequestID) { + return; + } + + if ( + response.error === + "The Access Key Id you provided does not exist in our records." + ) { + // Session expired. + window.location.reload(); + } else if (response.error === "Access Denied.") { + let pathPrefix = ""; + if (internalPaths) { + const decodedPath = decodeURLString(internalPaths); + pathPrefix = decodedPath.endsWith("/") + ? decodedPath + : decodedPath + "/"; + } + + const permitItems = permissionItems( + bucketName, + pathPrefix, + allowResources || [] + ); + + if (!permitItems || permitItems.length === 0) { + dispatch( + setErrorSnackMessage({ + errorMessage: response.error, + detailedError: response.error, + }) + ); + } else { + dispatch(setRecords(permitItems)); + } + + return; + } + + // This indicates final messages is received. + if (response.request_end) { + dispatch(setLoadingObjectsList(false)); + dispatch(setLoadingRecords(false)); + return; + } + + if (response.data) { + dispatch(newMessage(response.data)); + } + } + }, + [dispatch, internalPaths, allowResources, bucketName] + ); + + const initWSRequest = useCallback( + (path: string, date: Date) => { + if (objectsWS && objectsWS.readyState === 1) { + try { + const newRequestID = currentRequestID + 1; + dispatch(resetMessages()); + + const request: WebsocketRequest = { + bucket_name: bucketName, + prefix: encodeURLString(path), + mode: rewindEnabled || showDeleted ? "rewind" : "objects", + date: date.toISOString(), + request_id: newRequestID, + }; + + objectsWS.send(JSON.stringify(request)); + + // We store the new ID for the requestID + currentRequestID = newRequestID; + } catch (e) { + console.log(e); + } + } else { + // Socket is disconnected, we request reconnection but will need to recreate call + const dupRequest = () => { + initWSRequest(path, date); + }; + + initWSConnection(dupRequest); + } + }, + [bucketName, rewindEnabled, showDeleted, dispatch] + ); + + useEffect(() => { + return () => { + const request: WebsocketRequest = { + mode: "cancel", + request_id: currentRequestID, + }; + + if (objectsWS && objectsWS.readyState === 1) { + objectsWS.send(JSON.stringify(request)); + } + }; + }, []); + + useEffect(() => { + if (objectsWS?.readyState === 1) { + const decodedIPaths = decodeURLString(internalPaths); + + if (decodedIPaths.endsWith("/") || decodedIPaths === "") { + dispatch(setObjectDetailsView(false)); + dispatch(setSelectedObjectView(null)); + dispatch( + setSimplePathHandler(decodedIPaths === "" ? "/" : decodedIPaths) + ); + } else { + dispatch(setLoadingObjectInfo(true)); + dispatch(setObjectDetailsView(true)); + dispatch(setLoadingVersions(true)); + dispatch( + setSelectedObjectView( + `${decodedIPaths ? `${encodeURLString(decodedIPaths)}` : ``}` + ) + ); + dispatch( + setSimplePathHandler( + `${decodedIPaths.split("/").slice(0, -1).join("/")}/` + ) + ); + } + } + }, [internalPaths, rewindDate, rewindEnabled, dispatch]); + + // Direct file access effect / prefix + useEffect(() => { + if (!loadingObjects && loadRecords && !rewindEnabled) { + const parentPath = `${decodeURLString(internalPaths) + .split("/") + .slice(0, -1) + .join("/")}/`; + + initWSRequest(parentPath, new Date()); + } + }, [ + loadingObjects, + loadRecords, + bucketName, + bucketToRewind, + dispatch, + internalPaths, + rewindDate, + rewindEnabled, + initWSRequest, + detailsOpen, + ]); + + const displayListObjects = hasPermission(bucketName, [ + IAM_SCOPES.S3_LIST_BUCKET, + ]); + + // Common objects list + useEffect(() => { + // begin watch if bucketName in bucketList and start pressed + if (loadingObjects && displayListObjects) { + let pathPrefix = ""; + if (internalPaths) { + const decodedPath = decodeURLString(internalPaths); + + // internalPaths are selected (file details), we split and get parent folder + if (selectedInternalPaths === internalPaths) { + pathPrefix = `${decodeURLString(internalPaths) + .split("/") + .slice(0, -1) + .join("/")}/`; + } else { + pathPrefix = decodedPath.endsWith("/") + ? decodedPath + : decodedPath + "/"; + } + } + + let requestDate = new Date(); + + if (rewindEnabled && rewindDate) { + requestDate = rewindDate; + } + + initWSRequest(pathPrefix, requestDate); + } else { + dispatch(setLoadingObjectsList(false)); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [ + loadingObjects, + internalPaths, + dispatch, + rewindDate, + rewindEnabled, + displayListObjects, + initWSRequest, + ]); + useEffect(() => { dispatch(setVersionsModeEnabled({ status: false })); }, [internalPaths, dispatch]); + useEffect(() => { + if (loadingVersioning) { + if (displayListObjects) { + api + .invoke("GET", `/api/v1/buckets/${bucketName}/versioning`) + .then((res: BucketVersioning) => { + dispatch(setIsVersioned(res.is_versioned)); + dispatch(setLoadingVersioning(false)); + }) + .catch((err: ErrorResponseHandler) => { + console.error( + "Error Getting Object Versioning Status: ", + err.detailedError + ); + dispatch(setLoadingVersioning(false)); + }); + } else { + dispatch(setLoadingVersioning(false)); + dispatch(resetMessages()); + } + } + }, [bucketName, loadingVersioning, dispatch, displayListObjects]); + + useEffect(() => { + if (loadingLocking) { + if (displayListObjects) { + api + .invoke("GET", `/api/v1/buckets/${bucketName}/object-locking`) + .then((res: BucketObjectLocking) => { + dispatch(setLockingEnabled(res.object_locking_enabled)); + dispatch(setLoadingLocking(false)); + }) + .catch((err: ErrorResponseHandler) => { + console.error( + "Error Getting Object Locking Status: ", + err.detailedError + ); + dispatch(setLoadingLocking(false)); + }); + } else { + dispatch(resetMessages()); + dispatch(setLoadingLocking(false)); + } + } + }, [bucketName, loadingLocking, dispatch, displayListObjects]); + + useEffect(() => { + if (loadingLocking) { + if (displayListObjects) { + api + .invoke("GET", `/api/v1/buckets/${bucketName}/object-locking`) + .then((res: BucketObjectLocking) => { + dispatch(setLockingEnabled(res.object_locking_enabled)); + setLoadingLocking(false); + }) + .catch((err: ErrorResponseHandler) => { + console.error( + "Error Getting Object Locking Status: ", + err.detailedError + ); + setLoadingLocking(false); + }); + } else { + dispatch(resetMessages()); + setLoadingLocking(false); + } + } + }, [bucketName, loadingLocking, dispatch, displayListObjects]); + const openBucketConfiguration = () => { navigate(`/buckets/${bucketName}/admin`); }; diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/ListBuckets.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/ListBuckets.tsx index 63d5081fc..cafa428b4 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/ListBuckets.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/ListBuckets.tsx @@ -64,6 +64,7 @@ import { selFeatures } from "../../consoleSlice"; import AutoColorIcon from "../../Common/Components/AutoColorIcon"; import TooltipWrapper from "../../Common/TooltipWrapper/TooltipWrapper"; import AButton from "../../Common/AButton/AButton"; +import { setLoadingObjectsList } from "../../ObjectBrowser/objectBrowserSlice"; const styles = (theme: Theme) => createStyles({ @@ -123,6 +124,7 @@ const ListBuckets = ({ classes }: IListBucketsProps) => { .then((res: BucketList) => { setLoading(false); setRecords(res.buckets || []); + dispatch(setLoadingObjectsList(true)); }) .catch((err: ErrorResponseHandler) => { setLoading(false); diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/CreatePathModal.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/CreatePathModal.tsx index d81427e3b..40f1d6564 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/CreatePathModal.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/CreatePathModal.tsx @@ -27,7 +27,7 @@ import { formFieldStyles, modalStyleUtils, } from "../../../../Common/FormComponents/common/styleLibrary"; -import { connect } from "react-redux"; +import { connect, useSelector } from "react-redux"; import { encodeURLString } from "../../../../../../common/utils"; import { BucketObjectItem } from "./types"; @@ -41,7 +41,6 @@ interface ICreatePath { bucketName: string; folderName: string; onClose: () => any; - existingFiles: BucketObjectItem[]; simplePath: string | null; } @@ -57,7 +56,6 @@ const CreatePathModal = ({ bucketName, onClose, classes, - existingFiles, simplePath, }: ICreatePath) => { const dispatch = useAppDispatch(); @@ -67,6 +65,8 @@ const CreatePathModal = ({ const [isFormValid, setIsFormValid] = useState(false); const [currentPath, setCurrentPath] = useState(bucketName); + const records = useSelector((state: AppState) => state.objectBrowser.records); + useEffect(() => { if (simplePath) { const newPath = `${bucketName}${ @@ -91,7 +91,7 @@ const CreatePathModal = ({ const sharesName = (record: BucketObjectItem) => record.name === folderPath + pathUrl; - if (existingFiles.findIndex(sharesName) !== -1) { + if (records.findIndex(sharesName) !== -1) { dispatch( setModalErrorSnackMessage({ errorMessage: "Folder cannot have the same name as an existing file", diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx index ed4361fe7..89756fa10 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjects.tsx @@ -30,15 +30,10 @@ import { Button } from "mds"; import createStyles from "@mui/styles/createStyles"; import Grid from "@mui/material/Grid"; import get from "lodash/get"; -import { BucketObjectItem, BucketObjectItemsList } from "./types"; import api from "../../../../../../common/api"; -import TableWrapper, { - ItemActions, -} from "../../../../Common/TableWrapper/TableWrapper"; import { decodeURLString, encodeURLString, - getClientOS, niceBytesInt, } from "../../../../../../common/utils"; @@ -50,27 +45,16 @@ import { searchField, tableStyles, } from "../../../../Common/FormComponents/common/styleLibrary"; -import { Badge, Typography } from "@mui/material"; +import { Badge } from "@mui/material"; import BrowserBreadcrumbs from "../../../../ObjectBrowser/BrowserBreadcrumbs"; -import { - download, - extensionPreview, - permissionItems, - sortListObjects, -} from "../utils"; -import { - BucketInfo, - BucketObjectLocking, - BucketQuota, - BucketVersioning, -} from "../../../types"; +import { extensionPreview } from "../utils"; +import { BucketInfo, BucketQuota } from "../../../types"; import { ErrorResponseHandler } from "../../../../../../common/types"; import ScreenTitle from "../../../../Common/ScreenTitle/ScreenTitle"; import { AppState, useAppDispatch } from "../../../../../../store"; import PageLayout from "../../../../Common/Layout/PageLayout"; - import { IAM_SCOPES, permissionTooltipHelper, @@ -79,7 +63,6 @@ import { hasPermission, SecureComponent, } from "../../../../../../common/SecureComponent"; - import withSuspense from "../../../../Common/Components/withSuspense"; import { BucketsIcon, @@ -91,7 +74,6 @@ import UploadFilesButton from "../../UploadFilesButton"; import DetailsListPanel from "./DetailsListPanel"; import ObjectDetailPanel from "./ObjectDetailPanel"; import ActionsListSection from "./ActionsListSection"; -import { listModeColumns, rewindModeColumns } from "./ListObjectsHelpers"; import VersionsNavigator from "../ObjectDetails/VersionsNavigator"; import CheckboxWrapper from "../../../../Common/FormComponents/CheckboxWrapper/CheckboxWrapper"; @@ -111,16 +93,21 @@ import { completeObject, failObject, openList, + resetMessages, resetRewind, - setLoadingObjectInfo, + setDownloadRenameModal, setLoadingObjectsList, + setLoadingRecords, setLoadingVersions, setNewObject, setObjectDetailsView, + setPreviewOpen, setSearchObjects, + setSelectedObjects, setSelectedObjectView, + setSelectedPreview, + setShareFileModalOpen, setShowDeletedObjects, - setSimplePathHandler, setVersionsModeEnabled, updateProgress, } from "../../../../ObjectBrowser/objectBrowserSlice"; @@ -132,8 +119,13 @@ import { setBucketInfo, } from "../../../BucketDetails/bucketDetailsSlice"; import RenameLongFileName from "../../../../ObjectBrowser/RenameLongFilename"; -import { selFeatures } from "../../../../consoleSlice"; import TooltipWrapper from "../../../../Common/TooltipWrapper/TooltipWrapper"; +import ListObjectsTable from "./ListObjectsTable"; +import { + downloadSelected, + openPreview, + openShare, +} from "../../../../ObjectBrowser/objectBrowserThunks"; const HistoryIcon = React.lazy( () => import("../../../../../../icons/HistoryIcon") @@ -159,28 +151,6 @@ const PreviewFileModal = withSuspense( const useStyles = makeStyles((theme: Theme) => createStyles({ - browsePaper: { - border: 0, - height: "calc(100vh - 210px)", - "&.isEmbedded": { - height: "calc(100vh - 315px)", - }, - "&.actionsPanelOpen": { - minHeight: "100%", - }, - "@media (max-width: 800px)": { - width: 800, - }, - }, - "@global": { - ".rowLine:hover .iconFileElm": { - backgroundImage: "url(/images/ob_file_filled.svg)", - }, - ".rowLine:hover .iconFolderElm": { - backgroundImage: "url(/images/ob_folder_filled.svg)", - }, - }, - badgeOverlap: { "& .MuiBadge-badge": { top: 10, @@ -215,12 +185,8 @@ const useStyles = makeStyles((theme: Theme) => breadcrumbsContainer: { padding: "12px 14px 5px", }, - parentWrapper: { - "@media (max-width: 800px)": { - overflowX: "auto", - }, - }, fullContainer: { + position: "relative", "@media (max-width: 799px)": { width: 0, }, @@ -255,31 +221,6 @@ const acceptDnDStyle = { borderColor: "#00e676", }; -function useInterval(callback: any, delay: number) { - const savedCallback = useRef(null); - - // Remember the latest callback. - useEffect(() => { - savedCallback.current = callback; - }, [callback]); - - // Set up the interval. - useEffect(() => { - function tick() { - if (savedCallback !== undefined && savedCallback.current) { - savedCallback.current(); - } - } - - if (delay !== null) { - let id = setInterval(tick, delay); - return () => clearInterval(id); - } - }, [delay]); -} - -const defLoading = Loading...; - const ListObjects = () => { const classes = useStyles(); const dispatch = useAppDispatch(); @@ -290,9 +231,6 @@ const ListObjects = () => { const rewindEnabled = useSelector( (state: AppState) => state.objectBrowser.rewind.rewindEnabled ); - const rewindDate = useSelector( - (state: AppState) => state.objectBrowser.rewind.dateToRewind - ); const bucketToRewind = useSelector( (state: AppState) => state.objectBrowser.rewind.bucketToRewind ); @@ -300,9 +238,6 @@ const ListObjects = () => { (state: AppState) => state.objectBrowser.versionsMode ); - const searchObjects = useSelector( - (state: AppState) => state.objectBrowser.searchObjects - ); const showDeleted = useSelector( (state: AppState) => state.objectBrowser.showDeleted ); @@ -312,47 +247,41 @@ const ListObjects = () => { const selectedInternalPaths = useSelector( (state: AppState) => state.objectBrowser.selectedInternalPaths ); - const loading = useSelector( + const loadingObjects = useSelector( (state: AppState) => state.objectBrowser.loadingObjects ); const simplePath = useSelector( (state: AppState) => state.objectBrowser.simplePath ); - const loadingBucket = useSelector(selBucketDetailsLoading); - const bucketInfo = useSelector(selBucketDetailsInfo); - const allowResources = useSelector( - (state: AppState) => state.console.session.allowResources + const isVersioned = useSelector( + (state: AppState) => state.objectBrowser.isVersioned + ); + const lockingEnabled = useSelector( + (state: AppState) => state.objectBrowser.lockingEnabled + ); + const downloadRenameModal = useSelector( + (state: AppState) => state.objectBrowser.downloadRenameModal + ); + const selectedPreview = useSelector( + (state: AppState) => state.objectBrowser.selectedPreview + ); + const shareFileModalOpen = useSelector( + (state: AppState) => state.objectBrowser.shareFileModalOpen + ); + const previewOpen = useSelector( + (state: AppState) => state.objectBrowser.previewOpen ); - const features = useSelector(selFeatures); - const obOnly = !!features?.includes("object-browser-only"); + const loadingBucket = useSelector(selBucketDetailsLoading); + const bucketInfo = useSelector(selBucketDetailsInfo); - const [records, setRecords] = useState([]); const [deleteMultipleOpen, setDeleteMultipleOpen] = useState(false); - const [loadingStartTime, setLoadingStartTime] = useState(0); - const [loadingMessage, setLoadingMessage] = - useState(defLoading); - const [loadingVersioning, setLoadingVersioning] = useState(true); - const [isVersioned, setIsVersioned] = useState(false); - const [loadingLocking, setLoadingLocking] = useState(true); - const [lockingEnabled, setLockingEnabled] = useState(false); const [rewindSelect, setRewindSelect] = useState(false); - const [selectedObjects, setSelectedObjects] = useState([]); - const [previewOpen, setPreviewOpen] = useState(false); - const [selectedPreview, setSelectedPreview] = - useState(null); - const [shareFileModalOpen, setShareFileModalOpen] = useState(false); - const [sortDirection, setSortDirection] = useState< - "ASC" | "DESC" | undefined - >("ASC"); - const [currentSortField, setCurrentSortField] = useState("name"); const [iniLoad, setIniLoad] = useState(false); const [canShareFile, setCanShareFile] = useState(false); const [canPreviewFile, setCanPreviewFile] = useState(false); const [quota, setQuota] = useState(null); - const [downloadRenameModal, setDownloadRenameModal] = - useState(null); const pathSegment = location.pathname.split("/browse/"); @@ -379,6 +308,30 @@ const ListObjects = () => { true ); + const displayDeleteObject = hasPermission(bucketName, [ + IAM_SCOPES.S3_DELETE_OBJECT, + ]); + const selectedObjects = useSelector( + (state: AppState) => state.objectBrowser.selectedObjects + ); + + useEffect(() => { + dispatch(setSearchObjects("")); + dispatch(setLoadingObjectsList(true)); + dispatch(setSelectedObjects([])); + }, [simplePath, dispatch]); + + useEffect(() => { + if (rewindEnabled) { + if (bucketToRewind !== bucketName) { + dispatch(resetRewind()); + return; + } + } + }, [rewindEnabled, bucketToRewind, bucketName, dispatch]); + + // END OF WS HANDLERS + useEffect(() => { if (folderUpload.current !== null) { folderUpload.current.setAttribute("directory", ""); @@ -433,39 +386,14 @@ const ListObjects = () => { return; } - if (selectedObjects.length === 0 && selectedInternalPaths === null) { + if ( + selectedObjects.length === 0 && + selectedInternalPaths === null && + !loadingObjects + ) { dispatch(setObjectDetailsView(false)); } - }, [selectedObjects, selectedInternalPaths, dispatch]); - - const displayDeleteObject = hasPermission(bucketName, [ - IAM_SCOPES.S3_DELETE_OBJECT, - ]); - - const displayListObjects = hasPermission(bucketName, [ - IAM_SCOPES.S3_LIST_BUCKET, - ]); - - const updateMessage = () => { - let timeDelta = Date.now() - loadingStartTime; - - if (timeDelta / 1000 >= 6) { - setLoadingMessage( - - - This operation is taking longer than expected... ( - {Math.ceil(timeDelta / 1000)}s) - - - ); - } else if (timeDelta / 1000 >= 3) { - setLoadingMessage( - - This operation is taking longer than expected... - - ); - } - }; + }, [selectedObjects, selectedInternalPaths, dispatch, loadingObjects]); useEffect(() => { if (!iniLoad) { @@ -474,280 +402,6 @@ const ListObjects = () => { } }, [iniLoad, dispatch, setIniLoad]); - useInterval(() => { - // Your custom logic here - if (loading) { - updateMessage(); - } - }, 1000); - - useEffect(() => { - if (loadingVersioning) { - if (displayListObjects) { - api - .invoke("GET", `/api/v1/buckets/${bucketName}/versioning`) - .then((res: BucketVersioning) => { - setIsVersioned(res.is_versioned); - setLoadingVersioning(false); - }) - .catch((err: ErrorResponseHandler) => { - console.error( - "Error Getting Object Versioning Status: ", - err.detailedError - ); - setLoadingVersioning(false); - }); - } else { - setLoadingVersioning(false); - setRecords([]); - } - } - }, [bucketName, loadingVersioning, dispatch, displayListObjects]); - - useEffect(() => { - if (loadingLocking) { - if (displayListObjects) { - api - .invoke("GET", `/api/v1/buckets/${bucketName}/object-locking`) - .then((res: BucketObjectLocking) => { - setLockingEnabled(res.object_locking_enabled); - setLoadingLocking(false); - }) - .catch((err: ErrorResponseHandler) => { - console.error( - "Error Getting Object Locking Status: ", - err.detailedError - ); - setLoadingLocking(false); - }); - } else { - setRecords([]); - setLoadingLocking(false); - } - } - }, [bucketName, loadingLocking, dispatch, displayListObjects]); - - useEffect(() => { - const decodedIPaths = decodeURLString(internalPaths); - - if (decodedIPaths.endsWith("/") || decodedIPaths === "") { - dispatch(setObjectDetailsView(false)); - dispatch(setSelectedObjectView(null)); - dispatch( - setSimplePathHandler(decodedIPaths === "" ? "/" : decodedIPaths) - ); - } else { - dispatch(setLoadingObjectInfo(true)); - dispatch(setObjectDetailsView(true)); - dispatch(setLoadingVersions(true)); - dispatch( - setSelectedObjectView( - `${decodedIPaths ? `${encodeURLString(decodedIPaths)}` : ``}` - ) - ); - dispatch( - setSimplePathHandler( - `${decodedIPaths.split("/").slice(0, -1).join("/")}/` - ) - ); - } - }, [internalPaths, rewindDate, rewindEnabled, dispatch]); - - useEffect(() => { - dispatch(setSearchObjects("")); - dispatch(setLoadingObjectsList(true)); - setSelectedObjects([]); - }, [simplePath, dispatch, setSelectedObjects]); - - useEffect(() => { - if (loading) { - if (displayListObjects) { - let pathPrefix = ""; - if (internalPaths) { - const decodedPath = decodeURLString(internalPaths); - pathPrefix = decodedPath.endsWith("/") - ? decodedPath - : decodedPath + "/"; - } - - let currentTimestamp = Date.now(); - setLoadingStartTime(currentTimestamp); - setLoadingMessage(defLoading); - - // We get URL to look into - let urlTake = `/api/v1/buckets/${bucketName}/objects`; - - // Is rewind enabled?, we use Rewind API - if (rewindEnabled) { - if (bucketToRewind !== bucketName) { - dispatch(resetRewind()); - return; - } - - if (rewindDate) { - const rewindParsed = rewindDate.toISOString(); - - urlTake = `/api/v1/buckets/${bucketName}/rewind/${rewindParsed}`; - } - } else if (showDeleted) { - // Do we want to display deleted items too?, we use rewind to current time to show everything - const currDate = new Date(); - const currDateISO = currDate.toISOString(); - - urlTake = `/api/v1/buckets/${bucketName}/rewind/${currDateISO}`; - } - - api - .invoke( - "GET", - `${urlTake}${ - pathPrefix ? `?prefix=${encodeURLString(pathPrefix)}` : `` - }` - ) - .then((res: BucketObjectItemsList) => { - const records: BucketObjectItem[] = res.objects || []; - const folders: BucketObjectItem[] = []; - const files: BucketObjectItem[] = []; - - // We separate items between folders or files to display folders at the beginning always. - records.forEach((record) => { - // We omit files from the same path - if (record.name !== decodeURLString(internalPaths)) { - // this is a folder - if (record.name.endsWith("/")) { - folders.push(record); - } else { - // this is a file - files.push(record); - } - } - }); - - const recordsInElement = [...folders, ...files]; - - if (recordsInElement.length === 0 && pathPrefix !== "") { - let pathTest = `/api/v1/buckets/${bucketName}/objects${ - internalPaths ? `?prefix=${internalPaths}` : "" - }`; - - if (rewindEnabled) { - const rewindParsed = rewindDate.toISOString(); - - let pathPrefix = ""; - if (internalPaths) { - const decodedPath = decodeURLString(internalPaths); - pathPrefix = decodedPath.endsWith("/") - ? decodedPath - : decodedPath + "/"; - } - - pathTest = `/api/v1/buckets/${bucketName}/rewind/${rewindParsed}${ - pathPrefix ? `?prefix=${encodeURLString(pathPrefix)}` : `` - }`; - } - - api - .invoke("GET", pathTest) - .then((res: BucketObjectItemsList) => { - //It is a file since it has elements in the object, setting file flag and waiting for component mount - if (!res.objects) { - // It is a folder, we remove loader & set original results list - dispatch(setLoadingObjectsList(false)); - setRecords(recordsInElement); - } else { - // This code prevents the program from opening a file when a substring of that file is entered as a new folder. - // Previously, if there was a file test1.txt and the folder test was created with the same prefix, the program - // would open test1.txt instead - let found = false; - let pathPrefixChopped = pathPrefix.slice( - 0, - pathPrefix.length - 1 - ); - for (let i = 0; i < res.objects.length; i++) { - if (res.objects[i].name === pathPrefixChopped) { - found = true; - } - } - if ( - (res.objects.length === 1 && - res.objects[0].name.endsWith("/")) || - !found - ) { - // This is a folder, we set the original results list - setRecords(recordsInElement); - } else { - // This is a file. We change URL & Open file details view. - dispatch(setObjectDetailsView(true)); - dispatch(setSelectedObjectView(internalPaths)); - - // We split the selected object URL & remove the last item to fetch the files list for the parent folder - const parentPath = `${decodeURLString(internalPaths) - .split("/") - .slice(0, -1) - .join("/")}/`; - - api - .invoke( - "GET", - `${urlTake}${ - pathPrefix - ? `?prefix=${encodeURLString(parentPath)}` - : `` - }` - ) - .then((res: BucketObjectItemsList) => { - const records: BucketObjectItem[] = res.objects || []; - - setRecords(records); - }) - .catch(() => {}); - } - - dispatch(setLoadingObjectsList(false)); - } - }) - .catch((err: ErrorResponseHandler) => { - dispatch(setLoadingObjectsList(false)); - dispatch(setErrorSnackMessage(err)); - }); - } else { - setRecords(recordsInElement); - dispatch(setLoadingObjectsList(false)); - } - }) - .catch((err: ErrorResponseHandler) => { - const permitItems = permissionItems( - bucketName, - pathPrefix, - allowResources || [] - ); - - if (!permitItems || permitItems.length === 0) { - dispatch(setErrorSnackMessage(err)); - } else { - setRecords(permitItems); - } - - dispatch(setLoadingObjectsList(false)); - }); - } else { - dispatch(setLoadingObjectsList(false)); - } - } - }, [ - loading, - dispatch, - bucketName, - rewindEnabled, - rewindDate, - internalPaths, - bucketInfo, - showDeleted, - displayListObjects, - bucketToRewind, - allowResources, - ]); - // bucket info useEffect(() => { if (loadingBucket) { @@ -769,7 +423,7 @@ const ListObjects = () => { if (refresh) { dispatch(setSnackBarMessage(`Objects deleted successfully.`)); - setSelectedObjects([]); + dispatch(setSelectedObjects([])); dispatch(setLoadingObjectsList(true)); } }; @@ -794,73 +448,6 @@ const ListObjects = () => { e.target.value = ""; }; - const downloadObject = (object: BucketObjectItem) => { - const identityDownload = encodeURLString( - `${bucketName}-${object.name}-${new Date().getTime()}-${Math.random()}` - ); - - const ID = makeid(8); - - const downloadCall = download( - bucketName, - encodeURLString(object.name), - object.version_id, - object.size, - null, - ID, - (progress) => { - dispatch( - updateProgress({ - instanceID: identityDownload, - progress: progress, - }) - ); - }, - () => { - dispatch(completeObject(identityDownload)); - }, - (msg: string) => { - dispatch(failObject({ instanceID: identityDownload, msg })); - }, - () => { - dispatch(cancelObjectInList(identityDownload)); - } - ); - storeCallForObjectWithID(ID, downloadCall); - dispatch( - setNewObject({ - ID, - bucketName, - done: false, - instanceID: identityDownload, - percentage: 0, - prefix: object.name, - type: "download", - waitingForFile: true, - failed: false, - cancelled: false, - errorMessage: "", - }) - ); - }; - - const openPath = (idElement: string) => { - setSelectedObjects([]); - - const newPath = `/buckets/${bucketName}/browse${ - idElement ? `/${encodeURLString(idElement)}` : `` - }`; - navigate(newPath); - - dispatch(setObjectDetailsView(true)); - dispatch(setLoadingVersions(true)); - dispatch( - setSelectedObjectView( - `${idElement ? `${encodeURLString(idElement)}` : ``}` - ) - ); - }; - const uploadObject = useCallback( (files: File[], folderPath: string): void => { let pathPrefix = ""; @@ -1064,7 +651,7 @@ const ListObjects = () => { } // We force objects list reload after all promises were handled dispatch(setLoadingObjectsList(true)); - setSelectedObjects([]); + dispatch(setSelectedObjects([])); }); }; @@ -1110,140 +697,18 @@ const ListObjects = () => { [isDragActive, isDragAccept] ); - const openPreview = () => { - if (selectedObjects.length === 1) { - let fileObject: BucketObjectItem | undefined; - - const findFunction = (currValue: BucketObjectItem) => - selectedObjects.includes(currValue.name); - - fileObject = filteredRecords.find(findFunction); - - if (fileObject) { - setSelectedPreview(fileObject); - setPreviewOpen(true); - } - } - }; - - const openShare = () => { - if (selectedObjects.length === 1) { - let fileObject: BucketObjectItem | undefined; - - const findFunction = (currValue: BucketObjectItem) => - selectedObjects.includes(currValue.name); - - fileObject = filteredRecords.find(findFunction); - - if (fileObject) { - setSelectedPreview(fileObject); - setShareFileModalOpen(true); - } - } - }; - const closeShareModal = () => { - setShareFileModalOpen(false); - setSelectedPreview(null); + dispatch(setShareFileModalOpen(false)); + dispatch(setSelectedPreview(null)); }; - const filteredRecords = records.filter((b: BucketObjectItem) => { - if (searchObjects === "") { - return true; - } else { - const objectName = b.name.toLowerCase(); - if (objectName.indexOf(searchObjects.toLowerCase()) >= 0) { - return true; - } else { - return false; - } - } - }); - const rewindCloseModal = () => { setRewindSelect(false); }; const closePreviewWindow = () => { - setPreviewOpen(false); - setSelectedPreview(null); - }; - - const selectListObjects = (e: React.ChangeEvent) => { - const targetD = e.target; - const value = targetD.value; - const checked = targetD.checked; - - let elements: string[] = [...selectedObjects]; // We clone the selectedBuckets array - - if (checked) { - // If the user has checked this field we need to push this to selectedBucketsList - elements.push(value); - } else { - // User has unchecked this field, we need to remove it from the list - elements = elements.filter((element) => element !== value); - } - setSelectedObjects(elements); - dispatch(setSelectedObjectView(null)); - - return elements; - }; - - const sortChange = (sortData: any) => { - const newSortDirection = get(sortData, "sortDirection", "DESC"); - setCurrentSortField(sortData.sortBy); - setSortDirection(newSortDirection); - dispatch(setLoadingObjectsList(true)); - }; - - const plSelect = filteredRecords; - const sortASC = plSelect.sort(sortListObjects(currentSortField)); - - let payload: BucketObjectItem[] = []; - - if (sortDirection === "ASC") { - payload = sortASC; - } else { - payload = sortASC.reverse(); - } - - const selectAllItems = () => { - dispatch(setSelectedObjectView(null)); - - if (selectedObjects.length === payload.length) { - setSelectedObjects([]); - return; - } - - const elements = payload.map((item) => item.name); - setSelectedObjects(elements); - }; - - const downloadSelected = () => { - if (selectedObjects.length !== 0) { - let itemsToDownload: BucketObjectItem[] = []; - - const filterFunction = (currValue: BucketObjectItem) => - selectedObjects.includes(currValue.name); - - itemsToDownload = filteredRecords.filter(filterFunction); - - // I case just one element is selected, then we trigger download modal validation. - // We are going to enforce zip download when multiple files are selected - if (itemsToDownload.length === 1) { - if ( - itemsToDownload[0].name.length > 200 && - getClientOS().toLowerCase().includes("win") - ) { - setDownloadRenameModal(itemsToDownload[0]); - return; - } - } - - itemsToDownload.forEach((filteredItem) => { - downloadObject(filteredItem); - }); - } + dispatch(setPreviewOpen(false)); + dispatch(setSelectedPreview(null)); }; const onClosePanel = (forceRefresh: boolean) => { @@ -1268,7 +733,7 @@ const ListObjects = () => { } dispatch(setObjectDetailsView(false)); - setSelectedObjects([]); + dispatch(setSelectedObjects([])); if (forceRefresh) { dispatch(setLoadingObjectsList(true)); @@ -1276,28 +741,22 @@ const ListObjects = () => { }; const setDeletedAction = () => { + dispatch(resetMessages()); dispatch(setShowDeletedObjects(!showDeleted)); onClosePanel(true); }; const closeRenameModal = () => { - setDownloadRenameModal(null); + dispatch(setDownloadRenameModal(null)); }; - const tableActions: ItemActions[] = [ - { - type: "view", - label: "View", - onClick: openPath, - sendOnlyId: true, - }, - ]; - const multiActionButtons = [ { - action: downloadSelected, + action: () => { + dispatch(downloadSelected(bucketName)); + }, label: "Download", - disabled: !canDownload || selectedObjects.length === 0, + disabled: !canDownload || selectedObjects?.length === 0, icon: , tooltip: canDownload ? "Download Selected" @@ -1307,14 +766,18 @@ const ListObjects = () => { ), }, { - action: openShare, + action: () => { + dispatch(openShare()); + }, label: "Share", disabled: selectedObjects.length !== 1 || !canShareFile, icon: , tooltip: canShareFile ? "Share Selected File" : "Sharing unavailable", }, { - action: openPreview, + action: () => { + dispatch(openPreview()); + }, label: "Preview", disabled: selectedObjects.length !== 1 || !canPreviewFile, icon: , @@ -1484,6 +947,8 @@ const ListObjects = () => { if (versionsMode) { dispatch(setLoadingVersions(true)); } else { + dispatch(resetMessages()); + dispatch(setLoadingRecords(true)); dispatch(setLoadingObjectsList(true)); } }} @@ -1560,7 +1025,6 @@ const ListObjects = () => { @@ -1581,48 +1045,7 @@ const ListObjects = () => { hidePathButton={false} /> - { - if (payload[index]?.delete_flag) { - return "deleted"; - } - - return ""; - }} - parentClassName={classes.parentWrapper} - /> + )} diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjectsTable.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjectsTable.tsx new file mode 100644 index 000000000..041e979d7 --- /dev/null +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ListObjectsTable.tsx @@ -0,0 +1,246 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { listModeColumns, rewindModeColumns } from "./ListObjectsHelpers"; +import TableWrapper, { + ItemActions, +} from "../../../../Common/TableWrapper/TableWrapper"; +import React, { useState } from "react"; +import makeStyles from "@mui/styles/makeStyles"; +import { Theme } from "@mui/material/styles"; +import createStyles from "@mui/styles/createStyles"; +import { useSelector } from "react-redux"; +import { AppState, useAppDispatch } from "../../../../../../store"; +import { selFeatures } from "../../../../consoleSlice"; +import { encodeURLString } from "../../../../../../common/utils"; +import { + setLoadingObjectsList, + setLoadingVersions, + setObjectDetailsView, + setSelectedObjects, + setSelectedObjectView, +} from "../../../../ObjectBrowser/objectBrowserSlice"; +import { useNavigate, useParams } from "react-router-dom"; +import get from "lodash/get"; +import { sortListObjects } from "../utils"; +import { BucketObjectItem } from "./types"; +import { + IAM_SCOPES, + permissionTooltipHelper, +} from "../../../../../../common/SecureComponent/permissions"; +import { hasPermission } from "../../../../../../common/SecureComponent"; + +const useStyles = makeStyles((theme: Theme) => + createStyles({ + browsePaper: { + border: 0, + height: "calc(100vh - 290px)", + "&.isEmbedded": { + height: "calc(100vh - 315px)", + }, + "&.actionsPanelOpen": { + minHeight: "100%", + }, + "@media (max-width: 800px)": { + width: 800, + }, + }, + parentWrapper: { + position: "relative", + height: "calc(100% - 60px)", + "@media (max-width: 800px)": { + overflowX: "auto", + }, + }, + "@global": { + ".rowLine:hover .iconFileElm": { + backgroundImage: "url(/images/ob_file_filled.svg)", + }, + ".rowLine:hover .iconFolderElm": { + backgroundImage: "url(/images/ob_folder_filled.svg)", + }, + }, + }) +); + +const ListObjectsTable = () => { + const classes = useStyles(); + const dispatch = useAppDispatch(); + const params = useParams(); + const navigate = useNavigate(); + + const [sortDirection, setSortDirection] = useState< + "ASC" | "DESC" | undefined + >("ASC"); + const [currentSortField, setCurrentSortField] = useState("name"); + + const bucketName = params.bucketName || ""; + + const detailsOpen = useSelector( + (state: AppState) => state.objectBrowser.objectDetailsOpen + ); + + const loadingObjects = useSelector( + (state: AppState) => state.objectBrowser.loadingObjects + ); + + const features = useSelector(selFeatures); + const obOnly = !!features?.includes("object-browser-only"); + + const rewindEnabled = useSelector( + (state: AppState) => state.objectBrowser.rewind.rewindEnabled + ); + const records = useSelector((state: AppState) => state.objectBrowser.records); + const searchObjects = useSelector( + (state: AppState) => state.objectBrowser.searchObjects + ); + const selectedObjects = useSelector( + (state: AppState) => state.objectBrowser.selectedObjects + ); + + const displayListObjects = hasPermission(bucketName, [ + IAM_SCOPES.S3_LIST_BUCKET, + ]); + + const filteredRecords = records.filter((b: BucketObjectItem) => { + if (searchObjects === "") { + return true; + } else { + const objectName = b.name.toLowerCase(); + if (objectName.indexOf(searchObjects.toLowerCase()) >= 0) { + return true; + } else { + return false; + } + } + }); + + const plSelect = filteredRecords; + const sortASC = plSelect.sort(sortListObjects(currentSortField)); + + let payload: BucketObjectItem[] = []; + + if (sortDirection === "ASC") { + payload = sortASC; + } else { + payload = sortASC.reverse(); + } + + const openPath = (idElement: string) => { + dispatch(setSelectedObjects([])); + + const newPath = `/buckets/${bucketName}/browse${ + idElement ? `/${encodeURLString(idElement)}` : `` + }`; + navigate(newPath); + + dispatch(setObjectDetailsView(true)); + dispatch(setLoadingVersions(true)); + dispatch( + setSelectedObjectView( + `${idElement ? `${encodeURLString(idElement)}` : ``}` + ) + ); + }; + const tableActions: ItemActions[] = [ + { + type: "view", + label: "View", + onClick: openPath, + sendOnlyId: true, + }, + ]; + + const sortChange = (sortData: any) => { + const newSortDirection = get(sortData, "sortDirection", "DESC"); + setCurrentSortField(sortData.sortBy); + setSortDirection(newSortDirection); + dispatch(setLoadingObjectsList(true)); + }; + + const selectAllItems = () => { + dispatch(setSelectedObjectView(null)); + + if (selectedObjects.length === payload.length) { + dispatch(setSelectedObjects([])); + return; + } + + const elements = payload.map((item) => item.name); + dispatch(setSelectedObjects(elements)); + }; + + const selectListObjects = (e: React.ChangeEvent) => { + const targetD = e.target; + const value = targetD.value; + const checked = targetD.checked; + + let elements: string[] = [...selectedObjects]; // We clone the selectedBuckets array + + if (checked) { + // If the user has checked this field we need to push this to selectedBucketsList + elements.push(value); + } else { + // User has unchecked this field, we need to remove it from the list + elements = elements.filter((element) => element !== value); + } + dispatch(setSelectedObjects(elements)); + dispatch(setSelectedObjectView(null)); + + return elements; + }; + + return ( + { + if (payload[index]?.delete_flag) { + return "deleted"; + } + + return ""; + }} + parentClassName={classes.parentWrapper} + /> + ); +}; +export default ListObjectsTable; diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ObjectDetailPanel.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ObjectDetailPanel.tsx index 38855a96d..11c7a74dc 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ObjectDetailPanel.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/ObjectDetailPanel.tsx @@ -29,7 +29,7 @@ import { spacingUtils, textStyleUtils, } from "../../../../Common/FormComponents/common/styleLibrary"; -import { IFileInfo } from "../ObjectDetails/types"; +import { IFileInfo, MetadataResponse } from "../ObjectDetails/types"; import { download, extensionPreview } from "../utils"; import { ErrorResponseHandler } from "../../../../../../common/types"; @@ -189,6 +189,8 @@ const ObjectDetailPanel = ({ const [previewOpen, setPreviewOpen] = useState(false); const [totalVersionsSize, setTotalVersionsSize] = useState(0); const [longFileOpen, setLongFileOpen] = useState(false); + const [metaData, setMetaData] = useState(null); + const [loadMetadata, setLoadingMetadata] = useState(false); const internalPathsDecoded = decodeURLString(internalPaths) || ""; const allPathData = internalPathsDecoded.split("/"); @@ -212,6 +214,10 @@ const ObjectDetailPanel = ({ ) || emptyFile; } + if (!infoElement.is_delete_marker) { + setLoadingMetadata(true); + } + setActualInfo(infoElement); } }, [selectedVersion, distributedSetup, allInfoElements]); @@ -242,8 +248,14 @@ const ObjectDetailPanel = ({ setTotalVersionsSize(tVersionSize); } else { - setActualInfo(result[0]); + const resInfo = result[0]; + + setActualInfo(resInfo); setVersions([]); + + if (!resInfo.is_delete_marker) { + setLoadingMetadata(true); + } } dispatch(setLoadingObjectInfo(false)); @@ -262,6 +274,26 @@ const ObjectDetailPanel = ({ selectedVersion, ]); + useEffect(() => { + if (loadMetadata && internalPaths !== "") { + api + .invoke( + "GET", + `/api/v1/buckets/${bucketName}/objects/metadata?prefix=${internalPaths}` + ) + .then((res: MetadataResponse) => { + let metadata = get(res, "objectMetadata", {}); + + setMetaData(metadata); + setLoadingMetadata(false); + }) + .catch((err) => { + console.error("Error Getting Metadata Status: ", err.detailedError); + setLoadingMetadata(false); + }); + } + }, [bucketName, internalPaths, loadMetadata]); + let tagKeys: string[] = []; if (actualInfo && actualInfo.tags) { @@ -649,7 +681,7 @@ const ObjectDetailPanel = ({ version_id: actualInfo.version_id || "null", size: parseInt(actualInfo.size || "0"), content_type: "", - last_modified: new Date(actualInfo.last_modified), + last_modified: actualInfo.last_modified, }} onClosePreview={() => { setPreviewOpen(false); @@ -838,20 +870,19 @@ const ObjectDetailPanel = ({ - - Metadata - - - - {actualInfo ? ( - - ) : null} - + {!actualInfo.is_delete_marker && ( + + + Metadata + + + + {actualInfo && metaData ? ( + + ) : null} + + + )} )} diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/types.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/types.tsx index df6999e66..0ccd41445 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/types.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ListObjects/types.tsx @@ -14,17 +14,48 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +import { IFileInfo } from "../ObjectDetails/types"; + export interface BucketObjectItem { name: string; size: number; etag?: string; - last_modified: Date; + last_modified: string; content_type?: string; version_id: string; delete_flag?: boolean; + is_latest?: boolean; } export interface BucketObjectItemsList { objects: BucketObjectItem[]; total?: number; } + +export interface WebsocketRequest { + mode: "objects" | "rewind" | "close" | "cancel"; + bucket_name?: string; + prefix?: string; + date?: string; + request_id: number; +} + +export interface WebsocketResponse { + request_id: number; + error?: string; + request_end?: boolean; + data?: ObjectResponse[]; +} +export interface ObjectResponse { + name: string; + last_modified: string; + size: number; + version_id: string; + delete_flag: boolean; + is_latest: boolean; +} + +export interface IRestoreLocalObjectList { + prefix: string; + objectInfo: IFileInfo; +} diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/ObjectMetaData.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/ObjectMetaData.tsx index 89aa0e9fb..feaf81b87 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/ObjectMetaData.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/ObjectMetaData.tsx @@ -1,8 +1,21 @@ -import React, { Fragment, useCallback, useEffect, useState } from "react"; -import useApi from "../../../../Common/Hooks/useApi"; -import { ErrorResponseHandler } from "../../../../../../common/types"; -import { MetadataResponse } from "./types"; -import get from "lodash/get"; +// This file is part of MinIO Console Server +// Copyright (c) 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import React, { Fragment } from "react"; +import { withStyles } from "@mui/styles"; import Grid from "@mui/material/Grid"; import { Box, Table, TableBody, TableCell, TableRow } from "@mui/material"; import { Theme } from "@mui/material/styles"; @@ -11,13 +24,10 @@ import { detailsPanel, spacingUtils, } from "../../../../Common/FormComponents/common/styleLibrary"; -import { withStyles } from "@mui/styles"; interface IObjectMetadata { - bucketName: string; - internalPaths: string; + metaData: any; classes?: any; - actualInfo: any; linear?: boolean; } @@ -45,38 +55,11 @@ const styles = (theme: Theme) => }); const ObjectMetaData = ({ - bucketName, - internalPaths, + metaData, classes, - actualInfo, linear = false, }: IObjectMetadata) => { - const [metaData, setMetaData] = useState({}); - - const onMetaDataSuccess = (res: MetadataResponse) => { - let metadata = get(res, "objectMetadata", {}); - - setMetaData(metadata); - }; - const onMetaDataError = (err: ErrorResponseHandler) => false; - - const [, invokeMetaDataApi] = useApi(onMetaDataSuccess, onMetaDataError); - const metaKeys = Object.keys(metaData); - const loadMetaData = useCallback(() => { - invokeMetaDataApi( - "GET", - `/api/v1/buckets/${bucketName}/objects/metadata?prefix=${internalPaths}` - ); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [bucketName, internalPaths, actualInfo]); - - useEffect(() => { - if (actualInfo) { - loadMetaData(); - } - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [actualInfo, loadMetaData]); if (linear) { return ( diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/RestoreFileVersion.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/RestoreFileVersion.tsx index f07a9efca..451f6d057 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/RestoreFileVersion.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/RestoreFileVersion.tsx @@ -29,12 +29,14 @@ import ConfirmDialog from "../../../../Common/ModalWrapper/ConfirmDialog"; import RecoverIcon from "../../../../../../icons/RecoverIcon"; import { setErrorSnackMessage } from "../../../../../../systemSlice"; import { useAppDispatch } from "../../../../../../store"; +import { IFileInfo } from "./types"; +import { restoreLocalObjectList } from "../../../../ObjectBrowser/objectBrowserSlice"; interface IRestoreFileVersion { classes: any; restoreOpen: boolean; bucketName: string; - versionID: string; + versionToRestore: IFileInfo; objectPath: string; onCloseAndUpdate: (refresh: boolean) => void; } @@ -46,7 +48,7 @@ const styles = (theme: Theme) => const RestoreFileVersion = ({ classes, - versionID, + versionToRestore, bucketName, objectPath, restoreOpen, @@ -63,11 +65,18 @@ const RestoreFileVersion = ({ "PUT", `/api/v1/buckets/${bucketName}/objects/restore?prefix=${encodeURLString( objectPath - )}&version_id=${versionID}` + )}&version_id=${versionToRestore.version_id}` ) .then((res: any) => { + console.log("REStORE", res); setRestoreLoading(false); onCloseAndUpdate(true); + dispatch( + restoreLocalObjectList({ + prefix: objectPath, + objectInfo: versionToRestore, + }) + ); }) .catch((error: ErrorResponseHandler) => { dispatch(setErrorSnackMessage(error)); @@ -95,7 +104,7 @@ const RestoreFileVersion = ({ Are you sure you want to restore
{objectPath}
with Version ID:
- {versionID}? + {versionToRestore.version_id}? } /> diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx index 8a20d401a..2be434ebd 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/ObjectDetails/VersionsNavigator.tsx @@ -181,7 +181,7 @@ const VersionsNavigator = ({ const [objectToShare, setObjectToShare] = useState(null); const [versions, setVersions] = useState([]); const [restoreVersionOpen, setRestoreVersionOpen] = useState(false); - const [restoreVersion, setRestoreVersion] = useState(""); + const [restoreVersion, setRestoreVersion] = useState(null); const [sortValue, setSortValue] = useState("date"); const [previewOpen, setPreviewOpen] = useState(false); const [deleteNonCurrentOpen, setDeleteNonCurrentOpen] = @@ -313,7 +313,7 @@ const VersionsNavigator = ({ }; const onRestoreItem = (item: IFileInfo) => { - setRestoreVersion(item.version_id || ""); + setRestoreVersion(item); setRestoreVersionOpen(true); }; @@ -334,7 +334,7 @@ const VersionsNavigator = ({ const closeRestoreModal = (reloadObjectData: boolean) => { setRestoreVersionOpen(false); - setRestoreVersion(""); + setRestoreVersion(null); if (reloadObjectData) { dispatch(setLoadingVersions(true)); @@ -454,11 +454,11 @@ const VersionsNavigator = ({ dataObject={objectToShare || actualInfo} /> )} - {restoreVersionOpen && actualInfo && ( + {restoreVersionOpen && actualInfo && restoreVersion && ( @@ -477,7 +477,7 @@ const VersionsNavigator = ({ objectToShare && objectToShare.size ? objectToShare.size : "0" ), content_type: "", - last_modified: new Date(actualInfo.last_modified), + last_modified: actualInfo.last_modified, }} onClosePreview={() => { setPreviewOpen(false); @@ -514,7 +514,6 @@ const VersionsNavigator = ({ diff --git a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts index b7aa8ff68..e52590317 100644 --- a/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts +++ b/portal-ui/src/screens/Console/Buckets/ListBuckets/Objects/utils.ts @@ -247,7 +247,7 @@ export const permissionItems = ( returnElements.push({ name: `${currentElementInPath}/`, size: 0, - last_modified: new Date(), + last_modified: "", version_id: "", }); } @@ -276,7 +276,7 @@ export const permissionItems = ( pathToRouteElements.length > 0 ? "/" : "" }${splitElement}/`, size: 0, - last_modified: new Date(), + last_modified: "", version_id: "", }); return false; diff --git a/portal-ui/src/screens/Console/ObjectBrowser/BrowserBreadcrumbs.tsx b/portal-ui/src/screens/Console/ObjectBrowser/BrowserBreadcrumbs.tsx index 788899154..e7d5a7519 100644 --- a/portal-ui/src/screens/Console/ObjectBrowser/BrowserBreadcrumbs.tsx +++ b/portal-ui/src/screens/Console/ObjectBrowser/BrowserBreadcrumbs.tsx @@ -31,7 +31,6 @@ import { IAM_SCOPES, permissionTooltipHelper, } from "../../../common/SecureComponent/permissions"; -import { BucketObjectItem } from "../Buckets/ListBuckets/Objects/ListObjects/types"; import withSuspense from "../Common/Components/withSuspense"; import { setSnackBarMessage } from "../../../systemSlice"; import { AppState, useAppDispatch } from "../../../store"; @@ -58,7 +57,6 @@ interface IObjectBrowser { bucketName: string; internalPaths: string; hidePathButton?: boolean; - existingFiles: BucketObjectItem[]; additionalOptions?: React.ReactNode; } @@ -66,7 +64,6 @@ const BrowserBreadcrumbs = ({ classes, bucketName, internalPaths, - existingFiles, hidePathButton, additionalOptions, }: IObjectBrowser) => { @@ -176,7 +173,6 @@ const BrowserBreadcrumbs = ({ bucketName={bucketName} folderName={internalPaths} onClose={closeAddFolderModal} - existingFiles={existingFiles} /> )} diff --git a/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserSlice.ts b/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserSlice.ts index 72e819e2e..82d22235a 100644 --- a/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserSlice.ts +++ b/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserSlice.ts @@ -16,6 +16,10 @@ import { createSlice, PayloadAction } from "@reduxjs/toolkit"; import { IFileItem, ObjectBrowserState } from "./types"; +import { + BucketObjectItem, + IRestoreLocalObjectList, +} from "../Buckets/ListBuckets/Objects/ListObjects/types"; const defaultRewind = { rewindEnabled: false, @@ -47,6 +51,18 @@ const initialState: ObjectBrowserState = { showDeleted: false, selectedInternalPaths: null, simplePath: null, + // object browser + records: [], + loadRecords: true, + loadingVersioning: true, + isVersioned: false, + lockingEnabled: false, + loadingLocking: false, + selectedObjects: [], + downloadRenameModal: null, + selectedPreview: null, + previewOpen: false, + shareFileModalOpen: false, }; export const objectBrowserSlice = createSlice({ @@ -259,6 +275,67 @@ export const objectBrowserSlice = createSlice({ action.payload, ]; }, + setRecords: (state, action: PayloadAction) => { + state.records = action.payload; + }, + setLoadingVersioning: (state, action: PayloadAction) => { + state.loadingVersioning = action.payload; + }, + setIsVersioned: (state, action: PayloadAction) => { + state.isVersioned = action.payload; + }, + setLockingEnabled: (state, action: PayloadAction) => { + state.lockingEnabled = action.payload; + }, + setLoadingLocking: (state, action: PayloadAction) => { + state.loadingLocking = action.payload; + }, + newMessage: (state, action: PayloadAction) => { + state.records = [...state.records, ...action.payload]; + }, + resetMessages: (state) => { + state.records = []; + }, + setLoadingRecords: (state, action: PayloadAction) => { + state.loadRecords = action.payload; + }, + setSelectedObjects: (state, action: PayloadAction) => { + state.selectedObjects = action.payload; + }, + setDownloadRenameModal: ( + state, + action: PayloadAction + ) => { + state.downloadRenameModal = action.payload; + }, + setSelectedPreview: ( + state, + action: PayloadAction + ) => { + state.selectedPreview = action.payload; + }, + setPreviewOpen: (state, action: PayloadAction) => { + state.previewOpen = action.payload; + }, + setShareFileModalOpen: (state, action: PayloadAction) => { + state.shareFileModalOpen = action.payload; + }, + restoreLocalObjectList: ( + state, + action: PayloadAction + ) => { + const indexToReplace = state.records.findIndex( + (element) => element.name === action.payload.prefix + ); + + if (indexToReplace >= 0) { + state.records[indexToReplace].delete_flag = + action.payload.objectInfo.is_delete_marker; + state.records[indexToReplace].size = parseInt( + action.payload.objectInfo.size || "0" + ); + } + }, }, }); export const { @@ -287,6 +364,20 @@ export const { setSimplePathHandler, newDownloadInit, newUploadInit, + setRecords, + resetMessages, + setLoadingVersioning, + setIsVersioned, + setLoadingLocking, + setLockingEnabled, + newMessage, + setSelectedObjects, + setDownloadRenameModal, + setSelectedPreview, + setPreviewOpen, + setShareFileModalOpen, + setLoadingRecords, + restoreLocalObjectList, } = objectBrowserSlice.actions; export default objectBrowserSlice.reducer; diff --git a/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts b/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts new file mode 100644 index 000000000..b5367cc5d --- /dev/null +++ b/portal-ui/src/screens/Console/ObjectBrowser/objectBrowserThunks.ts @@ -0,0 +1,157 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +import { createAsyncThunk } from "@reduxjs/toolkit"; +import { AppState } from "../../../store"; +import { encodeURLString, getClientOS } from "../../../common/utils"; +import { BucketObjectItem } from "../Buckets/ListBuckets/Objects/ListObjects/types"; +import { makeid, storeCallForObjectWithID } from "./transferManager"; +import { download } from "../Buckets/ListBuckets/Objects/utils"; +import { + cancelObjectInList, + completeObject, + failObject, + setDownloadRenameModal, + setNewObject, + setPreviewOpen, + setSelectedPreview, + setShareFileModalOpen, + updateProgress, +} from "./objectBrowserSlice"; + +export const downloadSelected = createAsyncThunk( + "objectBrowser/downloadSelected", + async (bucketName: string, { getState, rejectWithValue, dispatch }) => { + const state = getState() as AppState; + + const downloadObject = (object: BucketObjectItem) => { + const identityDownload = encodeURLString( + `${bucketName}-${object.name}-${new Date().getTime()}-${Math.random()}` + ); + + const ID = makeid(8); + + const downloadCall = download( + bucketName, + encodeURLString(object.name), + object.version_id, + object.size, + null, + ID, + (progress) => { + dispatch( + updateProgress({ + instanceID: identityDownload, + progress: progress, + }) + ); + }, + () => { + dispatch(completeObject(identityDownload)); + }, + (msg: string) => { + dispatch(failObject({ instanceID: identityDownload, msg })); + }, + () => { + dispatch(cancelObjectInList(identityDownload)); + } + ); + storeCallForObjectWithID(ID, downloadCall); + dispatch( + setNewObject({ + ID, + bucketName, + done: false, + instanceID: identityDownload, + percentage: 0, + prefix: object.name, + type: "download", + waitingForFile: true, + failed: false, + cancelled: false, + errorMessage: "", + }) + ); + }; + + if (state.objectBrowser.selectedObjects.length !== 0) { + let itemsToDownload: BucketObjectItem[] = []; + + const filterFunction = (currValue: BucketObjectItem) => + state.objectBrowser.selectedObjects.includes(currValue.name); + + itemsToDownload = state.objectBrowser.records.filter(filterFunction); + + // I case just one element is selected, then we trigger download modal validation. + // We are going to enforce zip download when multiple files are selected + if (itemsToDownload.length === 1) { + if ( + itemsToDownload[0].name.length > 200 && + getClientOS().toLowerCase().includes("win") + ) { + dispatch(setDownloadRenameModal(itemsToDownload[0])); + return; + } + } + + itemsToDownload.forEach((filteredItem) => { + downloadObject(filteredItem); + }); + } + } +); + +export const openPreview = createAsyncThunk( + "objectBrowser/openPreview", + async (_, { getState, rejectWithValue, dispatch }) => { + const state = getState() as AppState; + + if (state.objectBrowser.selectedObjects.length === 1) { + let fileObject: BucketObjectItem | undefined; + + const findFunction = (currValue: BucketObjectItem) => + state.objectBrowser.selectedObjects.includes(currValue.name); + + fileObject = state.objectBrowser.records.find(findFunction); + + if (fileObject) { + dispatch(setSelectedPreview(fileObject)); + dispatch(setPreviewOpen(true)); + } + } + } +); + +export const openShare = createAsyncThunk( + "objectBrowser/openShare", + async (_, { getState, rejectWithValue, dispatch }) => { + const state = getState() as AppState; + + if (state.objectBrowser.selectedObjects.length === 1) { + let fileObject: BucketObjectItem | undefined; + + const findFunction = (currValue: BucketObjectItem) => + state.objectBrowser.selectedObjects.includes(currValue.name); + + fileObject = state.objectBrowser.records.find(findFunction); + + if (fileObject) { + dispatch(setSelectedPreview(fileObject)); + dispatch(setShareFileModalOpen(true)); + } + } + } +); diff --git a/portal-ui/src/screens/Console/ObjectBrowser/types.ts b/portal-ui/src/screens/Console/ObjectBrowser/types.ts index aef50857f..9151ca658 100644 --- a/portal-ui/src/screens/Console/ObjectBrowser/types.ts +++ b/portal-ui/src/screens/Console/ObjectBrowser/types.ts @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +import { BucketObjectItem } from "../Buckets/ListBuckets/Objects/ListObjects/types"; + export const REWIND_SET_ENABLE = "REWIND/SET_ENABLE"; export const REWIND_RESET_REWIND = "REWIND/RESET_REWIND"; @@ -76,6 +78,17 @@ export interface ObjectBrowserState { objectDetailsOpen: boolean; selectedInternalPaths: string | null; simplePath: string | null; + records: BucketObjectItem[]; + loadRecords: boolean; + loadingVersioning: boolean; + isVersioned: boolean; + lockingEnabled: boolean; + loadingLocking: boolean; + selectedObjects: string[]; + downloadRenameModal: BucketObjectItem | null; + selectedPreview: BucketObjectItem | null; + previewOpen: boolean; + shareFileModalOpen: boolean; } export interface ObjectManager { diff --git a/restapi/admin_objects.go b/restapi/admin_objects.go new file mode 100644 index 000000000..1d5097ba8 --- /dev/null +++ b/restapi/admin_objects.go @@ -0,0 +1,102 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package restapi + +import ( + "context" + "encoding/base64" + "time" + + "github.com/minio/mc/cmd" + "github.com/minio/minio-go/v7" +) + +type objectsListOpts struct { + BucketName string + Prefix string + Date time.Time +} + +type ObjectsRequest struct { + Mode string `json:"mode,nonempty"` + BucketName string `json:"bucket_name"` + Prefix string `json:"prefix"` + Date string `json:"date"` + RequestID int64 `json:"request_id"` +} + +type WSResponse struct { + RequestID int64 `json:"request_id,nonempty"` + Error string `json:"error,omitempty"` + RequestEnd bool `json:"request_end,omitempty"` + Data []ObjectResponse `json:"data,omitempty"` +} + +type ObjectResponse struct { + Name string `json:"name,nonempty"` + LastModified string `json:"last_modified,nonempty"` + Size int64 `json:"size,nonempty"` + VersionID string `json:"version_id,nonempty"` + DeleteMarker bool `json:"delete_flag,omitempty"` + IsLatest bool `json:"is_latest,omitempty"` +} + +func getObjectsOptionsFromReq(request ObjectsRequest) (*objectsListOpts, error) { + pOptions := objectsListOpts{ + BucketName: request.BucketName, + Prefix: "", + } + + prefix := request.Prefix + + if prefix != "" { + encodedPrefix := SanitizeEncodedPrefix(prefix) + decodedPrefix, err := base64.StdEncoding.DecodeString(encodedPrefix) + if err != nil { + LogError("error decoding prefix: %v", err) + return nil, err + } + + pOptions.Prefix = string(decodedPrefix) + } + + if request.Mode == "rewind" { + parsedDate, errDate := time.Parse(time.RFC3339, request.Date) + + if errDate != nil { + return nil, errDate + } + + pOptions.Date = parsedDate + } + + return &pOptions, nil +} + +func startObjectsListing(ctx context.Context, client MinioClient, objOpts *objectsListOpts) <-chan minio.ObjectInfo { + opts := minio.ListObjectsOptions{ + Prefix: objOpts.Prefix, + } + + return client.listObjects(ctx, objOpts.BucketName, opts) +} + +func startRewindListing(ctx context.Context, client MCClient, objOpts *objectsListOpts) <-chan *cmd.ClientContent { + lsRewind := client.list(ctx, cmd.ListOptions{TimeRef: objOpts.Date, WithDeleteMarkers: true}) + + return lsRewind +} diff --git a/restapi/admin_objects_test.go b/restapi/admin_objects_test.go new file mode 100644 index 000000000..f1e0b5e58 --- /dev/null +++ b/restapi/admin_objects_test.go @@ -0,0 +1,236 @@ +// This file is part of MinIO Console Server +// Copyright (c) 2022 MinIO, Inc. +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package restapi + +import ( + "context" + "testing" + "time" + + mc "github.com/minio/mc/cmd" + "github.com/minio/minio-go/v7" + "github.com/stretchr/testify/assert" +) + +func TestWSRewindObjects(t *testing.T) { + assert := assert.New(t) + client := s3ClientMock{} + + tests := []struct { + name string + testOptions objectsListOpts + testMessages []*mc.ClientContent + }{ + { + name: "Get list with multiple elements", + testOptions: objectsListOpts{ + BucketName: "buckettest", + Prefix: "/", + Date: time.Now(), + }, + testMessages: []*mc.ClientContent{ + { + BucketName: "buckettest", + URL: mc.ClientURL{Path: "/file1.txt"}, + }, + { + BucketName: "buckettest", + URL: mc.ClientURL{Path: "/file2.txt"}, + }, + { + BucketName: "buckettest", + URL: mc.ClientURL{Path: "/path1"}, + }, + }, + }, + { + name: "Empty list of elements", + testOptions: objectsListOpts{ + BucketName: "emptybucket", + Prefix: "/", + Date: time.Now(), + }, + testMessages: []*mc.ClientContent{}, + }, + { + name: "Get list with one element", + testOptions: objectsListOpts{ + BucketName: "buckettest", + Prefix: "/", + Date: time.Now(), + }, + testMessages: []*mc.ClientContent{ + { + BucketName: "buckettestsingle", + URL: mc.ClientURL{Path: "/file12.txt"}, + }, + }, + }, + { + name: "Get data from subpaths", + testOptions: objectsListOpts{ + BucketName: "buckettest", + Prefix: "/path1/path2", + Date: time.Now(), + }, + testMessages: []*mc.ClientContent{ + { + BucketName: "buckettestsingle", + URL: mc.ClientURL{Path: "/path1/path2/file12.txt"}, + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + mcListMock = func(ctx context.Context, opts mc.ListOptions) <-chan *mc.ClientContent { + ch := make(chan *mc.ClientContent) + go func() { + defer close(ch) + for _, m := range tt.testMessages { + ch <- m + } + }() + return ch + } + + rewindList := startRewindListing(ctx, client, &tt.testOptions) + + // check that the rewindList got the same number of data from Console. + + totalItems := 0 + for data := range rewindList { + // Compare elements as we are defining the channel responses + assert.Equal(tt.testMessages[totalItems].URL.Path, data.URL.Path) + totalItems++ + } + assert.Equal(len(tt.testMessages), totalItems) + }) + } +} + +func TestWSListObjects(t *testing.T) { + assert := assert.New(t) + client := minioClientMock{} + + tests := []struct { + name string + wantErr bool + testOptions objectsListOpts + testMessages []minio.ObjectInfo + }{ + { + name: "Get list with multiple elements", + wantErr: false, + testOptions: objectsListOpts{ + BucketName: "buckettest", + Prefix: "/", + }, + testMessages: []minio.ObjectInfo{ + { + Key: "/file1.txt", + Size: 500, + IsLatest: true, + LastModified: time.Now(), + }, + { + Key: "/file2.txt", + Size: 500, + IsLatest: true, + LastModified: time.Now(), + }, + { + Key: "/path1", + }, + }, + }, + { + name: "Empty list of elements", + wantErr: false, + testOptions: objectsListOpts{ + BucketName: "emptybucket", + Prefix: "/", + }, + testMessages: []minio.ObjectInfo{}, + }, + { + name: "Get list with one element", + wantErr: false, + testOptions: objectsListOpts{ + BucketName: "buckettest", + Prefix: "/", + }, + testMessages: []minio.ObjectInfo{ + { + Key: "/file2.txt", + Size: 500, + IsLatest: true, + LastModified: time.Now(), + }, + }, + }, + { + name: "Get data from subpaths", + wantErr: false, + testOptions: objectsListOpts{ + BucketName: "buckettest", + Prefix: "/path1/path2", + }, + testMessages: []minio.ObjectInfo{ + { + Key: "/path1/path2/file1.txt", + Size: 500, + IsLatest: true, + LastModified: time.Now(), + }, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + minioListObjectsMock = func(ctx context.Context, bucket string, opts minio.ListObjectsOptions) <-chan minio.ObjectInfo { + ch := make(chan minio.ObjectInfo) + go func() { + defer close(ch) + for _, m := range tt.testMessages { + ch <- m + } + }() + return ch + } + + objectsListing := startObjectsListing(ctx, client, &tt.testOptions) + + // check that the TestReceiver got the same number of data from Console + totalItems := 0 + for data := range objectsListing { + // Compare elements as we are defining the channel responses + assert.Equal(tt.testMessages[totalItems].Key, data.Key) + totalItems++ + } + assert.Equal(len(tt.testMessages), totalItems) + }) + } +} diff --git a/restapi/embedded_spec.go b/restapi/embedded_spec.go index a6172188b..812483616 100644 --- a/restapi/embedded_spec.go +++ b/restapi/embedded_spec.go @@ -7159,6 +7159,9 @@ func init() { "delete_flag": { "type": "boolean" }, + "is_latest": { + "type": "boolean" + }, "last_modified": { "type": "string" }, @@ -15464,6 +15467,9 @@ func init() { "delete_flag": { "type": "boolean" }, + "is_latest": { + "type": "boolean" + }, "last_modified": { "type": "string" }, diff --git a/restapi/user_objects.go b/restapi/user_objects.go index 1f5036e1a..8d3cc8e77 100644 --- a/restapi/user_objects.go +++ b/restapi/user_objects.go @@ -225,6 +225,7 @@ func listBucketObjects(ctx context.Context, client MinioClient, bucketName strin Recursive: recursive, WithVersions: withVersions, WithMetadata: withMetadata, + MaxKeys: 100, } if withMetadata { opts.MaxKeys = 1 diff --git a/restapi/ws_handle.go b/restapi/ws_handle.go index 9c7849d88..9dcb09bf3 100644 --- a/restapi/ws_handle.go +++ b/restapi/ws_handle.go @@ -18,20 +18,23 @@ package restapi import ( "context" + "encoding/json" "fmt" + "log" "net" "net/http" "strconv" "strings" "time" + "github.com/minio/madmin-go/v2" + "github.com/minio/console/pkg/utils" "github.com/go-openapi/errors" "github.com/gorilla/websocket" "github.com/minio/console/models" "github.com/minio/console/pkg/auth" - "github.com/minio/madmin-go/v2" ) var upgrader = websocket.Upgrader{ @@ -70,6 +73,18 @@ type wsS3Client struct { client MCClient } +// ConsoleWebSocketMClient interface of a Websocket Client +type ConsoleWebsocketMClient interface { + objectManager(options objectsListOpts) +} + +type wsMinioClient struct { + // websocket connection. + conn wsConn + // MinIO admin Client + client minioClient +} + // WSConn interface with all functions to be implemented // by mock when testing, it should include all websocket.Conn // respective api calls that are used within this project. @@ -135,9 +150,9 @@ func serveWS(w http.ResponseWriter, req *http.Request) { } // Un-comment for development so websockets work on port 5005 - // upgrader.CheckOrigin = func(r *http.Request) bool { - // return true - // } + /*upgrader.CheckOrigin = func(r *http.Request) bool { + return true + }*/ // upgrades the HTTP server connection to the WebSocket protocol. conn, err := upgrader.Upgrade(w, req, nil) @@ -236,7 +251,7 @@ func serveWS(w http.ResponseWriter, req *http.Request) { closeWsConn(conn) return } - wsS3Client, err := newWebSocketS3Client(conn, session, wOptions.BucketName) + wsS3Client, err := newWebSocketS3Client(conn, session, wOptions.BucketName, "") if err != nil { ErrorWithContext(ctx, err) closeWsConn(conn) @@ -272,6 +287,15 @@ func serveWS(w http.ResponseWriter, req *http.Request) { } go wsAdminClient.profile(ctx, pOptions) + case strings.HasPrefix(wsPath, `/objectManager`): + wsMinioClient, err := newWebSocketMinioClient(conn, session) + if err != nil { + ErrorWithContext(ctx, err) + closeWsConn(conn) + return + } + + go wsMinioClient.objectManager(session) default: // path not found closeWsConn(conn) @@ -299,10 +323,10 @@ func newWebSocketAdminClient(conn *websocket.Conn, autClaims *models.Principal) } // newWebSocketS3Client returns a wsAdminClient authenticated as Console admin -func newWebSocketS3Client(conn *websocket.Conn, claims *models.Principal, bucketName string) (*wsS3Client, error) { +func newWebSocketS3Client(conn *websocket.Conn, claims *models.Principal, bucketName, prefix string) (*wsS3Client, error) { // Only start Websocket Interaction after user has been // authenticated with MinIO - s3Client, err := newS3BucketClient(claims, bucketName, "") + s3Client, err := newS3BucketClient(claims, bucketName, prefix) if err != nil { LogError("error creating S3Client:", err) return nil, err @@ -318,6 +342,28 @@ func newWebSocketS3Client(conn *websocket.Conn, claims *models.Principal, bucket return wsS3Client, nil } +func newWebSocketMinioClient(conn *websocket.Conn, claims *models.Principal) (*wsMinioClient, error) { + // Only start Websocket Interaction after user has been + // authenticated with MinIO + + mClient, err := newMinioClient(claims) + if err != nil { + LogError("error creating MinioClient:", err) + return nil, err + } + + // create a websocket connection interface implementation + // defining the connection to be used + wsConnection := wsConn{conn: conn} + // create a minioClient interface implementation + // defining the client to be used + minioClient := minioClient{client: mClient} + + // create websocket client and handle request + wsMinioClient := &wsMinioClient{conn: wsConnection, client: minioClient} + return wsMinioClient, nil +} + // wsReadClientCtx reads the messages that come from the client // if the client sends a Close Message the context will be // canceled. If the connection is closed the goroutine inside @@ -492,6 +538,228 @@ func (wsc *wsAdminClient) profile(ctx context.Context, opts *profileOptions) { sendWsCloseMessage(wsc.conn, err) } +func (wsc *wsMinioClient) objectManager(session *models.Principal) { + // Storage of Cancel Contexts for this connection + cancelContexts := make(map[int64]context.CancelFunc) + + // Initial goroutine + defer func() { + // We close socket at the end of requests + wsc.conn.close() + for _, c := range cancelContexts { + // invoke cancel + c() + } + }() + + writeChannel := make(chan WSResponse) + done := make(chan interface{}) + + // Read goroutine + go func() { + for { + mType, message, err := wsc.conn.readMessage() + if err != nil { + LogInfo("Error while reading objectManager message", err) + close(done) + return + } + + if mType == websocket.TextMessage { + // We get request data & review information + var messageRequest ObjectsRequest + + err := json.Unmarshal(message, &messageRequest) + if err != nil { + LogInfo("Error on message request unmarshal") + close(done) + return + } + + // new message, new context + ctx, cancel := context.WithCancel(context.Background()) + + // We store the cancel func associated with this request + cancelContexts[messageRequest.RequestID] = cancel + + const itemsPerBatch = 1000 + switch messageRequest.Mode { + case "close": + close(done) + return + case "cancel": + // if we have that request id, cancel it + if cancelFunc, ok := cancelContexts[messageRequest.RequestID]; ok { + cancelFunc() + delete(cancelContexts, messageRequest.RequestID) + } + case "objects": + // cancel all previous open objects requests for listing + for rid, c := range cancelContexts { + if rid < messageRequest.RequestID { + // invoke cancel + c() + } + } + + // start listing and writing to web socket + go func() { + defer func() { + log.Println("Closing listing goroutine:", messageRequest.RequestID) + }() + objectRqConfigs, err := getObjectsOptionsFromReq(messageRequest) + if err != nil { + LogInfo(fmt.Sprintf("Error during Objects OptionsParse %s", err.Error())) + return + } + var buffer []ObjectResponse + for lsObj := range startObjectsListing(ctx, wsc.client, objectRqConfigs) { + if cancelContexts[messageRequest.RequestID] == nil { + return + } + if lsObj.Err != nil { + writeChannel <- WSResponse{ + RequestID: messageRequest.RequestID, + Error: lsObj.Err.Error(), + } + + continue + } + objItem := ObjectResponse{ + Name: lsObj.Key, + Size: lsObj.Size, + LastModified: lsObj.LastModified.Format(time.RFC3339), + VersionID: lsObj.VersionID, + IsLatest: lsObj.IsLatest, + DeleteMarker: lsObj.IsDeleteMarker, + } + buffer = append(buffer, objItem) + + if len(buffer) >= itemsPerBatch { + writeChannel <- WSResponse{ + RequestID: messageRequest.RequestID, + Data: buffer, + } + buffer = nil + } + } + if len(buffer) > 0 { + writeChannel <- WSResponse{ + RequestID: messageRequest.RequestID, + Data: buffer, + } + } + + writeChannel <- WSResponse{ + RequestID: messageRequest.RequestID, + RequestEnd: true, + } + + // remove the cancellation context + delete(cancelContexts, messageRequest.RequestID) + }() + case "rewind": + // cancel all previous open objects requests for listing + for rid, c := range cancelContexts { + if rid < messageRequest.RequestID { + // invoke cancel + c() + } + } + + // start listing and writing to web socket + go func() { + objectRqConfigs, err := getObjectsOptionsFromReq(messageRequest) + if err != nil { + LogInfo(fmt.Sprintf("Error during Objects OptionsParse %s", err.Error())) + cancel() + return + } + + s3Client, err := newS3BucketClient(session, objectRqConfigs.BucketName, objectRqConfigs.Prefix) + if err != nil { + LogError("error creating S3Client:", err) + close(done) + cancel() + return + } + + mcS3C := mcClient{client: s3Client} + + var buffer []ObjectResponse + + for lsObj := range startRewindListing(ctx, mcS3C, objectRqConfigs) { + if lsObj.Err != nil { + writeChannel <- WSResponse{ + RequestID: messageRequest.RequestID, + Error: lsObj.Err.String(), + } + + continue + } + + name := strings.ReplaceAll(lsObj.URL.Path, fmt.Sprintf("/%s/", objectRqConfigs.BucketName), "") + + objItem := ObjectResponse{ + Name: name, + Size: lsObj.Size, + LastModified: lsObj.Time.Format(time.RFC3339), + VersionID: lsObj.VersionID, + IsLatest: lsObj.IsLatest, + DeleteMarker: lsObj.IsDeleteMarker, + } + buffer = append(buffer, objItem) + + if len(buffer) >= itemsPerBatch { + writeChannel <- WSResponse{ + RequestID: messageRequest.RequestID, + Data: buffer, + } + buffer = nil + } + + } + if len(buffer) > 0 { + writeChannel <- WSResponse{ + RequestID: messageRequest.RequestID, + Data: buffer, + } + } + + writeChannel <- WSResponse{ + RequestID: messageRequest.RequestID, + RequestEnd: true, + } + + // remove the cancellation context + delete(cancelContexts, messageRequest.RequestID) + }() + } + } + } + }() + + // Write goroutine + go func() { + for writeM := range writeChannel { + jsonData, err := json.Marshal(writeM) + if err != nil { + LogInfo("Error while parsing the response", err) + return + } + + err = wsc.conn.writeMessage(websocket.TextMessage, jsonData) + + if err != nil { + LogInfo("Error while writing the message", err) + return + } + } + }() + + <-done +} + // sendWsCloseMessage sends Websocket Connection Close Message indicating the Status Code // see https://tools.ietf.org/html/rfc6455#page-45 func sendWsCloseMessage(conn WSConn, err error) { @@ -507,7 +775,7 @@ func sendWsCloseMessage(conn WSConn, err error) { return } // else, internal server error - conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, ErrDefault.Error())) + conn.writeMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseInternalServerErr, err.Error())) return } // normal closure diff --git a/swagger-console.yml b/swagger-console.yml index 932152635..358d34ad8 100644 --- a/swagger-console.yml +++ b/swagger-console.yml @@ -5203,6 +5203,8 @@ definitions: type: string name: type: string + is_latest: + type: boolean rewindResponse: type: object