From 7e9f5562528ba1fac241459f8daeaf9550c19eaa Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Mon, 14 Sep 2026 11:19:54 -0700 Subject: [PATCH] feat: add webui concurrent uploads with progress tracking Queue uploads with a concurrency limit, track per-file and aggregate progress, and preserve batch bucket/prefix context. Limit concurrent uploads to three. Display aggregate byte progress as a percentage and bytesDone / bytesTotal, completion and failure totals, and the number of active uploads in the message box. Display Queued, Uploading, and Failed files with individual progress bars; completed uploads remain in the aggregate totals without filling the live file list. Allow queued and active uploads to be cancelled from the progress widget. Remove queued uploads immediately, abort active object and multipart part requests, and abort a multipart upload on the server after its upload ID is available. --- webui/web/explorer.html | 280 +++++++++++++++++++++++++++++++--------- webui/web/js/api.js | 12 +- 2 files changed, 225 insertions(+), 67 deletions(-) diff --git a/webui/web/explorer.html b/webui/web/explorer.html index f0572c1e..7ce8be5a 100644 --- a/webui/web/explorer.html +++ b/webui/web/explorer.html @@ -2969,35 +2969,194 @@ under the License. // overall byte progress across every file/part of the current upload // batch, so a large file no longer looks stalled with no feedback. let uploadProgressEl = null; + let nextUploadBatchId = 1; + let activeUploadCount = 0; + const uploadQueue = []; + const uploadBatches = new Map(); + const MAX_CONCURRENT_UPLOADS = 3; + + function getUploadStatusLabel(file) { + if (file.state === 'completed') return 'Completed'; + if (file.state === 'failed') return 'Failed'; + if (file.state === 'cancelled') return 'Cancelled'; + if (file.state === 'cancelling') return 'Cancelling'; + if (file.state === 'finalizing') return 'Finalizing'; + if (file.state === 'uploading') return 'Uploading'; + return 'Queued'; + } + + function renderUploadProgress() { + if (uploadBatches.size === 0) { + if (uploadProgressEl) { + uploadProgressEl.remove(); + uploadProgressEl = null; + } + return; + } - function showUploadProgress(fileName, fileIndex, fileCount, bytesDone, bytesTotal) { if (!uploadProgressEl) { uploadProgressEl = document.createElement('div'); uploadProgressEl.id = 'upload-progress-indicator'; - uploadProgressEl.className = 'fixed bottom-4 right-4 z-50 w-80 bg-white rounded-lg shadow-lg border border-gray-200 p-4'; + uploadProgressEl.className = 'fixed bottom-4 right-4 z-50 w-96 max-w-[calc(100vw-2rem)] bg-white rounded-lg shadow-lg border border-gray-200 p-4'; + uploadProgressEl.setAttribute('role', 'status'); + uploadProgressEl.setAttribute('aria-live', 'polite'); + const handleCancel = event => { + const button = event.target.closest('[data-cancel-upload]'); + if (!button) return; + if (event.type === 'pointerdown') event.preventDefault(); + cancelUpload(Number(button.dataset.batchId), Number(button.dataset.fileIndex)); + }; + uploadProgressEl.addEventListener('pointerdown', handleCancel); + uploadProgressEl.addEventListener('click', handleCancel); document.body.appendChild(uploadProgressEl); } + const files = Array.from(uploadBatches.values()).flatMap(batch => batch.files); + const trackedFiles = files.filter(file => file.state !== 'cancelled'); + const bytesDone = trackedFiles.reduce((sum, file) => sum + file.bytesDone, 0); + const bytesTotal = trackedFiles.reduce((sum, file) => sum + file.size, 0); const percent = bytesTotal > 0 ? Math.min(100, Math.round((bytesDone / bytesTotal) * 100)) : 0; - const label = fileCount > 1 ? `File ${fileIndex} of ${fileCount}: ${fileName}` : fileName; + const completedCount = trackedFiles.filter(file => file.state === 'completed').length; + const failedCount = trackedFiles.filter(file => file.state === 'failed').length; + const cancelledCount = files.filter(file => file.state === 'cancelled').length; + const activeCount = files.filter(file => file.state === 'uploading' || file.state === 'cancelling' || file.state === 'finalizing').length; + const statusText = [ + `${completedCount} of ${trackedFiles.length} completed`, + failedCount > 0 ? `${failedCount} failed` : null, + cancelledCount > 0 ? `${cancelledCount} cancelled` : null + ].filter(Boolean).join(', '); uploadProgressEl.innerHTML = `
-

${escapeHtml(label)}

- ${percent}% +

Uploads${activeCount > 0 ? ` (${activeCount} active)` : ''}

+ ${percent}%
-
-
+
+
+
+

${escapeHtml(statusText)} · ${formatSize(bytesDone)} / ${formatSize(bytesTotal)}

+
+ ${files.filter(file => file.state !== 'completed' && file.state !== 'cancelled').map(file => { + const filePercent = file.size > 0 ? Math.min(100, Math.round((file.bytesDone / file.size) * 100)) : 0; + const statusLabel = getUploadStatusLabel(file); + return ` +
+
+ ${escapeHtml(file.name)} +
+ ${statusLabel} + ${file.state === 'queued' || file.state === 'uploading' || file.state === 'finalizing' ? `` : ''} +
+
+
+
+
+
+ `; + }).join('')}
-

${formatSize(bytesDone)} / ${formatSize(bytesTotal)}

`; + } - function hideUploadProgress() { - if (uploadProgressEl) { - uploadProgressEl.remove(); - uploadProgressEl = null; + function removeFinishedUploadBatches() { + if (uploadQueue.length > 0 || activeUploadCount > 0) return; + uploadBatches.clear(); + renderUploadProgress(); + } + + function updateUploadFileProgress(file, bytesDone) { + file.bytesDone = Math.max(file.bytesDone, Math.min(bytesDone, file.size)); + renderUploadProgress(); + } + + function finishUploadBatch(batch) { + if (batch.pendingFiles > 0) return; + + const successfulFiles = batch.files.filter(file => file.state === 'completed').length; + const failedFiles = batch.files.filter(file => file.state === 'failed').length; + if (successfulFiles > 0) { + showToast(`Uploaded ${successfulFiles} file${successfulFiles > 1 ? 's' : ''}${failedFiles > 0 ? ` (${failedFiles} failed)` : ''}`, 'success'); + if (currentBucket === batch.bucket && currentPrefix === batch.prefix) { + loadObjects(); + } } + batch.resolve(); + removeFinishedUploadBatches(); + } + + function cancelUpload(batchId, fileIndex) { + const batch = uploadBatches.get(batchId); + const file = batch?.files[fileIndex]; + if (!file || (file.state !== 'queued' && file.state !== 'uploading' && file.state !== 'finalizing')) return; + + if (file.state === 'queued') { + const queueIndex = uploadQueue.findIndex(item => item.file === file); + if (queueIndex !== -1) uploadQueue.splice(queueIndex, 1); + file.state = 'cancelled'; + batch.pendingFiles--; + showToast(`Cancelled upload: ${file.name}`, 'info'); + renderUploadProgress(); + finishUploadBatch(batch); + drainUploadQueue(); + return; + } + + file.state = 'cancelling'; + file.controller?.abort(); + renderUploadProgress(); + } + + async function runUploadFile(file, batch) { + file.state = 'uploading'; + file.controller = new AbortController(); + renderUploadProgress(); + + try { + if (file.size >= MULTIPART_THRESHOLD) { + await uploadMultipart(file.file, batch.bucket, file.key, (bytesDone) => { + updateUploadFileProgress(file, bytesDone); + }, () => { + file.state = 'finalizing'; + renderUploadProgress(); + }, file.controller.signal); + } else { + await api.putObject(batch.bucket, file.key, file.file, null, file.controller.signal); + updateUploadFileProgress(file, file.size); + } + if (file.state === 'cancelling') { + file.state = 'cancelled'; + } else { + file.bytesDone = file.size; + file.state = 'completed'; + } + } catch (error) { + if (file.state === 'cancelling' || error.name === 'AbortError') { + file.state = 'cancelled'; + showToast(`Cancelled upload: ${file.name}`, 'info'); + } else { + file.state = 'failed'; + file.error = error; + console.error('Upload error:', error); + showToast(`Failed to upload ${file.name}: ${error.message}`, 'error'); + } + } finally { + file.controller = null; + batch.pendingFiles--; + activeUploadCount--; + renderUploadProgress(); + finishUploadBatch(batch); + drainUploadQueue(); + } + } + + function drainUploadQueue() { + while (activeUploadCount < MAX_CONCURRENT_UPLOADS && uploadQueue.length > 0) { + const item = uploadQueue.shift(); + activeUploadCount++; + runUploadFile(item.file, item.batch); + } + removeFinishedUploadBatches(); } function openUploadDialog() { @@ -3036,88 +3195,83 @@ under the License. } async function uploadFiles(files) { - let successCount = 0; - let failCount = 0; + const uploadBucket = currentBucket; + const uploadPrefix = currentPrefix; + const batchId = nextUploadBatchId++; + const batchFiles = files.map((file, index) => ({ + file, + batchId, + fileIndex: index, + name: file.name, + key: uploadPrefix + file.name, + size: file.size, + bytesDone: 0, + state: 'queued', + error: null, + controller: null + })); + const batch = { + id: batchId, + bucket: uploadBucket, + prefix: uploadPrefix, + files: batchFiles, + pendingFiles: batchFiles.length, + resolve: null + }; + const completion = new Promise(resolve => { + batch.resolve = resolve; + }); + uploadBatches.set(batchId, batch); // Show uploading toast for multiple files if (files.length > 1) { showToast(`Uploading ${files.length} files...`, 'info'); } - const totalBytes = files.reduce((sum, f) => sum + f.size, 0); - let bytesDoneBeforeCurrentFile = 0; - - try { - for (let i = 0; i < files.length; i++) { - const file = files[i]; - const key = currentPrefix + file.name; - - try { - if (file.size >= MULTIPART_THRESHOLD) { - // Use multipart upload for large files - await uploadMultipart(file, key, (bytesDoneInFile) => { - showUploadProgress(file.name, i + 1, files.length, bytesDoneBeforeCurrentFile + bytesDoneInFile, totalBytes); - }); - } else { - // Use simple PUT for small files (no per-chunk hook to report progress - // through, so report it in one step once the PUT actually resolves — - // matches the multipart path only firing onProgress after each part is - // confirmed uploaded, never optimistically before) - await api.putObject(currentBucket, key, file); - showUploadProgress(file.name, i + 1, files.length, bytesDoneBeforeCurrentFile + file.size, totalBytes); - } - bytesDoneBeforeCurrentFile += file.size; - successCount++; - } catch (error) { - console.error('Upload error:', error); - showToast(`Failed to upload ${file.name}: ${error.message}`, 'error'); - failCount++; - } - } - } finally { - hideUploadProgress(); - } - - if (successCount > 0) { - showToast(`Uploaded ${successCount} file${successCount > 1 ? 's' : ''}${failCount > 0 ? ` (${failCount} failed)` : ''}`, 'success'); - loadObjects(); - } + batchFiles.forEach(file => uploadQueue.push({ file, batch })); + renderUploadProgress(); + drainUploadQueue(); + return completion; } - async function uploadMultipart(file, key, onProgress) { + async function uploadMultipart(file, bucket, key, onProgress, onFinalizing, signal) { const contentType = file.type || 'application/octet-stream'; // Calculate optimal part size for this file const partSize = calculatePartSize(file.size); console.log(`Uploading ${file.name}: ${formatSize(file.size)}, part size: ${formatSize(partSize)}`); - // Initiate multipart upload - const uploadId = await api.createMultipartUpload(currentBucket, key, contentType); - const parts = []; const totalParts = Math.ceil(file.size / partSize); + let uploadId; try { + uploadId = await api.createMultipartUpload(bucket, key, contentType, signal); + if (signal?.aborted) throw new DOMException('Upload cancelled', 'AbortError'); for (let partNumber = 1; partNumber <= totalParts; partNumber++) { const start = (partNumber - 1) * partSize; const end = Math.min(start + partSize, file.size); const chunk = file.slice(start, end); const chunkBuffer = await chunk.arrayBuffer(); - const etag = await api.uploadPart(currentBucket, key, uploadId, partNumber, chunkBuffer); + const etag = await api.uploadPart(bucket, key, uploadId, partNumber, chunkBuffer, signal); parts.push({ partNumber, etag }); if (onProgress) onProgress(end); } // Complete the multipart upload - await api.completeMultipartUpload(currentBucket, key, uploadId, parts); + if (signal?.aborted) throw new DOMException('Upload cancelled', 'AbortError'); + if (onFinalizing) onFinalizing(); + await api.completeMultipartUpload(bucket, key, uploadId, parts, signal); } catch (error) { // Abort the multipart upload on failure - try { - await api.abortMultipartUpload(currentBucket, key, uploadId); - } catch (abortError) { - console.error('Failed to abort multipart upload:', abortError); + if (uploadId) { + try { + await api.abortMultipartUpload(bucket, key, uploadId); + } catch (abortError) { + console.error('Failed to abort multipart upload:', abortError); + } } throw error; } @@ -3407,7 +3561,7 @@ under the License. if (bytes === 0) return '0 B'; const k = 1024; const sizes = ['B', 'KB', 'MB', 'GB', 'TB']; - const i = Math.floor(Math.log(bytes) / Math.log(k)); + const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length - 1); return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i]; } diff --git a/webui/web/js/api.js b/webui/web/js/api.js index ce8a6075..9ca59121 100644 --- a/webui/web/js/api.js +++ b/webui/web/js/api.js @@ -1395,7 +1395,7 @@ class VersityAPI { /** * Upload an object (PutObject) - for small files < 5MB */ - async putObject(bucket, key, file, contentType = null) { + async putObject(bucket, key, file, contentType = null, signal = undefined) { const finalContentType = contentType || file.type || 'application/octet-stream'; const path = `/${bucket}/${encodeS3Key(key)}`; @@ -1408,6 +1408,7 @@ class VersityAPI { method: 'PUT', headers: signed.headers, body: file, + signal, }); if (!response.ok) { @@ -1585,12 +1586,13 @@ class VersityAPI { * Initiate a multipart upload * Returns uploadId needed for subsequent parts */ - async createMultipartUpload(bucket, key, contentType = 'application/octet-stream') { + async createMultipartUpload(bucket, key, contentType = 'application/octet-stream', signal = undefined) { const fetchParams = await this.buildFetchParams('POST', `/${bucket}/${encodeS3Key(key)}`, { uploads: '' }, '', false, contentType); const response = await fetch(fetchParams.url, { method: 'POST', headers: fetchParams.headers, + signal, }); if (!response.ok) { @@ -1614,7 +1616,7 @@ class VersityAPI { * Upload a single part of a multipart upload * Returns ETag needed for CompleteMultipartUpload */ - async uploadPart(bucket, key, uploadId, partNumber, data) { + async uploadPart(bucket, key, uploadId, partNumber, data, signal = undefined) { const arrayBuffer = data instanceof ArrayBuffer ? data : await data.arrayBuffer(); const path = `/${bucket}/${encodeS3Key(key)}`; const queryParams = { @@ -1635,6 +1637,7 @@ class VersityAPI { method: 'PUT', headers: signed.headers, body: arrayBuffer, + signal, }); if (!response.ok) { @@ -1669,7 +1672,7 @@ class VersityAPI { * Complete a multipart upload * parts should be an array of { partNumber, etag } */ - async completeMultipartUpload(bucket, key, uploadId, parts) { + async completeMultipartUpload(bucket, key, uploadId, parts, signal = undefined) { let body = '\n'; // Sort parts by partNumber @@ -1690,6 +1693,7 @@ class VersityAPI { method: 'POST', headers: fetchParams.headers, body: body, + signal, }); if (!response.ok) {