mirror of
https://github.com/versity/versitygw.git
synced 2026-09-25 01:14:22 +00:00
feat: add webui concurrent uploads with progress tracking
Queue uploads with a concurrency limit, track per-file and aggregate progress, and preserve batch bucket/prefix context. Limit concurrent uploads to three. Display aggregate byte progress as a percentage and bytesDone / bytesTotal, completion and failure totals, and the number of active uploads in the message box. Display Queued, Uploading, and Failed files with individual progress bars; completed uploads remain in the aggregate totals without filling the live file list. Allow queued and active uploads to be cancelled from the progress widget. Remove queued uploads immediately, abort active object and multipart part requests, and abort a multipart upload on the server after its upload ID is available.
This commit is contained in:
Ben McClelland
+217
-63
@@ -2969,35 +2969,194 @@ under the License.
|
||||
// overall byte progress across every file/part of the current upload
|
||||
// batch, so a large file no longer looks stalled with no feedback.
|
||||
let uploadProgressEl = null;
|
||||
let nextUploadBatchId = 1;
|
||||
let activeUploadCount = 0;
|
||||
const uploadQueue = [];
|
||||
const uploadBatches = new Map();
|
||||
const MAX_CONCURRENT_UPLOADS = 3;
|
||||
|
||||
function getUploadStatusLabel(file) {
|
||||
if (file.state === 'completed') return 'Completed';
|
||||
if (file.state === 'failed') return 'Failed';
|
||||
if (file.state === 'cancelled') return 'Cancelled';
|
||||
if (file.state === 'cancelling') return 'Cancelling';
|
||||
if (file.state === 'finalizing') return 'Finalizing';
|
||||
if (file.state === 'uploading') return 'Uploading';
|
||||
return 'Queued';
|
||||
}
|
||||
|
||||
function renderUploadProgress() {
|
||||
if (uploadBatches.size === 0) {
|
||||
if (uploadProgressEl) {
|
||||
uploadProgressEl.remove();
|
||||
uploadProgressEl = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
function showUploadProgress(fileName, fileIndex, fileCount, bytesDone, bytesTotal) {
|
||||
if (!uploadProgressEl) {
|
||||
uploadProgressEl = document.createElement('div');
|
||||
uploadProgressEl.id = 'upload-progress-indicator';
|
||||
uploadProgressEl.className = 'fixed bottom-4 right-4 z-50 w-80 bg-white rounded-lg shadow-lg border border-gray-200 p-4';
|
||||
uploadProgressEl.className = 'fixed bottom-4 right-4 z-50 w-96 max-w-[calc(100vw-2rem)] bg-white rounded-lg shadow-lg border border-gray-200 p-4';
|
||||
uploadProgressEl.setAttribute('role', 'status');
|
||||
uploadProgressEl.setAttribute('aria-live', 'polite');
|
||||
const handleCancel = event => {
|
||||
const button = event.target.closest('[data-cancel-upload]');
|
||||
if (!button) return;
|
||||
if (event.type === 'pointerdown') event.preventDefault();
|
||||
cancelUpload(Number(button.dataset.batchId), Number(button.dataset.fileIndex));
|
||||
};
|
||||
uploadProgressEl.addEventListener('pointerdown', handleCancel);
|
||||
uploadProgressEl.addEventListener('click', handleCancel);
|
||||
document.body.appendChild(uploadProgressEl);
|
||||
}
|
||||
|
||||
const files = Array.from(uploadBatches.values()).flatMap(batch => batch.files);
|
||||
const trackedFiles = files.filter(file => file.state !== 'cancelled');
|
||||
const bytesDone = trackedFiles.reduce((sum, file) => sum + file.bytesDone, 0);
|
||||
const bytesTotal = trackedFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
const percent = bytesTotal > 0 ? Math.min(100, Math.round((bytesDone / bytesTotal) * 100)) : 0;
|
||||
const label = fileCount > 1 ? `File ${fileIndex} of ${fileCount}: ${fileName}` : fileName;
|
||||
const completedCount = trackedFiles.filter(file => file.state === 'completed').length;
|
||||
const failedCount = trackedFiles.filter(file => file.state === 'failed').length;
|
||||
const cancelledCount = files.filter(file => file.state === 'cancelled').length;
|
||||
const activeCount = files.filter(file => file.state === 'uploading' || file.state === 'cancelling' || file.state === 'finalizing').length;
|
||||
const statusText = [
|
||||
`${completedCount} of ${trackedFiles.length} completed`,
|
||||
failedCount > 0 ? `${failedCount} failed` : null,
|
||||
cancelledCount > 0 ? `${cancelledCount} cancelled` : null
|
||||
].filter(Boolean).join(', ');
|
||||
|
||||
uploadProgressEl.innerHTML = `
|
||||
<div class="flex items-center justify-between mb-2 gap-2">
|
||||
<p class="text-sm font-medium text-gray-800 truncate" title="${escapeHtml(fileName)}">${escapeHtml(label)}</p>
|
||||
<span class="text-sm font-semibold text-blue-600 shrink-0">${percent}%</span>
|
||||
<p class="text-sm font-medium text-gray-800">Uploads${activeCount > 0 ? ` (${activeCount} active)` : ''}</p>
|
||||
<span class="text-sm font-semibold text-primary shrink-0">${percent}%</span>
|
||||
</div>
|
||||
<div class="w-full bg-gray-200 rounded-full h-2 overflow-hidden">
|
||||
<div class="bg-blue-600 h-2 rounded-full transition-all duration-150" style="width: ${percent}%"></div>
|
||||
<div class="w-full bg-gray-200 rounded-full h-2 overflow-hidden" role="progressbar" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${percent}" aria-label="Upload progress">
|
||||
<div class="bg-primary h-2 rounded-full transition-all duration-150" style="width: ${percent}%"></div>
|
||||
</div>
|
||||
<p class="text-xs text-gray-500 mt-1">${escapeHtml(statusText)} · ${formatSize(bytesDone)} / ${formatSize(bytesTotal)}</p>
|
||||
<div class="mt-3 space-y-2 max-h-40 overflow-y-auto">
|
||||
${files.filter(file => file.state !== 'completed' && file.state !== 'cancelled').map(file => {
|
||||
const filePercent = file.size > 0 ? Math.min(100, Math.round((file.bytesDone / file.size) * 100)) : 0;
|
||||
const statusLabel = getUploadStatusLabel(file);
|
||||
return `
|
||||
<div class="text-xs" aria-label="${escapeHtml(file.name)}: ${statusLabel}">
|
||||
<div class="flex items-center justify-between gap-2">
|
||||
<span class="truncate text-gray-700" title="${escapeHtml(file.name)}">${escapeHtml(file.name)}</span>
|
||||
<div class="flex items-center gap-2 shrink-0">
|
||||
<span class="${file.state === 'failed' ? 'text-red-600' : 'text-gray-500'}">${statusLabel}</span>
|
||||
${file.state === 'queued' || file.state === 'uploading' || file.state === 'finalizing' ? `<button type="button" data-cancel-upload data-batch-id="${file.batchId}" data-file-index="${file.fileIndex}" class="text-red-600 hover:text-red-700 font-medium" aria-label="Cancel ${escapeHtml(file.name)}">Cancel</button>` : ''}
|
||||
</div>
|
||||
</div>
|
||||
<div class="w-full bg-gray-100 rounded-full h-1 mt-1 overflow-hidden">
|
||||
<div class="${file.state === 'failed' ? 'bg-red-400' : 'bg-primary'} h-1 rounded-full" style="width: ${filePercent}%"></div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('')}
|
||||
</div>
|
||||
<p class="text-xs text-gray-500 mt-1">${formatSize(bytesDone)} / ${formatSize(bytesTotal)}</p>
|
||||
`;
|
||||
|
||||
}
|
||||
|
||||
function hideUploadProgress() {
|
||||
if (uploadProgressEl) {
|
||||
uploadProgressEl.remove();
|
||||
uploadProgressEl = null;
|
||||
function removeFinishedUploadBatches() {
|
||||
if (uploadQueue.length > 0 || activeUploadCount > 0) return;
|
||||
uploadBatches.clear();
|
||||
renderUploadProgress();
|
||||
}
|
||||
|
||||
function updateUploadFileProgress(file, bytesDone) {
|
||||
file.bytesDone = Math.max(file.bytesDone, Math.min(bytesDone, file.size));
|
||||
renderUploadProgress();
|
||||
}
|
||||
|
||||
function finishUploadBatch(batch) {
|
||||
if (batch.pendingFiles > 0) return;
|
||||
|
||||
const successfulFiles = batch.files.filter(file => file.state === 'completed').length;
|
||||
const failedFiles = batch.files.filter(file => file.state === 'failed').length;
|
||||
if (successfulFiles > 0) {
|
||||
showToast(`Uploaded ${successfulFiles} file${successfulFiles > 1 ? 's' : ''}${failedFiles > 0 ? ` (${failedFiles} failed)` : ''}`, 'success');
|
||||
if (currentBucket === batch.bucket && currentPrefix === batch.prefix) {
|
||||
loadObjects();
|
||||
}
|
||||
}
|
||||
batch.resolve();
|
||||
removeFinishedUploadBatches();
|
||||
}
|
||||
|
||||
function cancelUpload(batchId, fileIndex) {
|
||||
const batch = uploadBatches.get(batchId);
|
||||
const file = batch?.files[fileIndex];
|
||||
if (!file || (file.state !== 'queued' && file.state !== 'uploading' && file.state !== 'finalizing')) return;
|
||||
|
||||
if (file.state === 'queued') {
|
||||
const queueIndex = uploadQueue.findIndex(item => item.file === file);
|
||||
if (queueIndex !== -1) uploadQueue.splice(queueIndex, 1);
|
||||
file.state = 'cancelled';
|
||||
batch.pendingFiles--;
|
||||
showToast(`Cancelled upload: ${file.name}`, 'info');
|
||||
renderUploadProgress();
|
||||
finishUploadBatch(batch);
|
||||
drainUploadQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
file.state = 'cancelling';
|
||||
file.controller?.abort();
|
||||
renderUploadProgress();
|
||||
}
|
||||
|
||||
async function runUploadFile(file, batch) {
|
||||
file.state = 'uploading';
|
||||
file.controller = new AbortController();
|
||||
renderUploadProgress();
|
||||
|
||||
try {
|
||||
if (file.size >= MULTIPART_THRESHOLD) {
|
||||
await uploadMultipart(file.file, batch.bucket, file.key, (bytesDone) => {
|
||||
updateUploadFileProgress(file, bytesDone);
|
||||
}, () => {
|
||||
file.state = 'finalizing';
|
||||
renderUploadProgress();
|
||||
}, file.controller.signal);
|
||||
} else {
|
||||
await api.putObject(batch.bucket, file.key, file.file, null, file.controller.signal);
|
||||
updateUploadFileProgress(file, file.size);
|
||||
}
|
||||
if (file.state === 'cancelling') {
|
||||
file.state = 'cancelled';
|
||||
} else {
|
||||
file.bytesDone = file.size;
|
||||
file.state = 'completed';
|
||||
}
|
||||
} catch (error) {
|
||||
if (file.state === 'cancelling' || error.name === 'AbortError') {
|
||||
file.state = 'cancelled';
|
||||
showToast(`Cancelled upload: ${file.name}`, 'info');
|
||||
} else {
|
||||
file.state = 'failed';
|
||||
file.error = error;
|
||||
console.error('Upload error:', error);
|
||||
showToast(`Failed to upload ${file.name}: ${error.message}`, 'error');
|
||||
}
|
||||
} finally {
|
||||
file.controller = null;
|
||||
batch.pendingFiles--;
|
||||
activeUploadCount--;
|
||||
renderUploadProgress();
|
||||
finishUploadBatch(batch);
|
||||
drainUploadQueue();
|
||||
}
|
||||
}
|
||||
|
||||
function drainUploadQueue() {
|
||||
while (activeUploadCount < MAX_CONCURRENT_UPLOADS && uploadQueue.length > 0) {
|
||||
const item = uploadQueue.shift();
|
||||
activeUploadCount++;
|
||||
runUploadFile(item.file, item.batch);
|
||||
}
|
||||
removeFinishedUploadBatches();
|
||||
}
|
||||
|
||||
function openUploadDialog() {
|
||||
@@ -3036,88 +3195,83 @@ under the License.
|
||||
}
|
||||
|
||||
async function uploadFiles(files) {
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
const uploadBucket = currentBucket;
|
||||
const uploadPrefix = currentPrefix;
|
||||
const batchId = nextUploadBatchId++;
|
||||
const batchFiles = files.map((file, index) => ({
|
||||
file,
|
||||
batchId,
|
||||
fileIndex: index,
|
||||
name: file.name,
|
||||
key: uploadPrefix + file.name,
|
||||
size: file.size,
|
||||
bytesDone: 0,
|
||||
state: 'queued',
|
||||
error: null,
|
||||
controller: null
|
||||
}));
|
||||
const batch = {
|
||||
id: batchId,
|
||||
bucket: uploadBucket,
|
||||
prefix: uploadPrefix,
|
||||
files: batchFiles,
|
||||
pendingFiles: batchFiles.length,
|
||||
resolve: null
|
||||
};
|
||||
const completion = new Promise(resolve => {
|
||||
batch.resolve = resolve;
|
||||
});
|
||||
uploadBatches.set(batchId, batch);
|
||||
|
||||
// Show uploading toast for multiple files
|
||||
if (files.length > 1) {
|
||||
showToast(`Uploading ${files.length} files...`, 'info');
|
||||
}
|
||||
|
||||
const totalBytes = files.reduce((sum, f) => sum + f.size, 0);
|
||||
let bytesDoneBeforeCurrentFile = 0;
|
||||
|
||||
try {
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
const key = currentPrefix + file.name;
|
||||
|
||||
try {
|
||||
if (file.size >= MULTIPART_THRESHOLD) {
|
||||
// Use multipart upload for large files
|
||||
await uploadMultipart(file, key, (bytesDoneInFile) => {
|
||||
showUploadProgress(file.name, i + 1, files.length, bytesDoneBeforeCurrentFile + bytesDoneInFile, totalBytes);
|
||||
});
|
||||
} else {
|
||||
// Use simple PUT for small files (no per-chunk hook to report progress
|
||||
// through, so report it in one step once the PUT actually resolves —
|
||||
// matches the multipart path only firing onProgress after each part is
|
||||
// confirmed uploaded, never optimistically before)
|
||||
await api.putObject(currentBucket, key, file);
|
||||
showUploadProgress(file.name, i + 1, files.length, bytesDoneBeforeCurrentFile + file.size, totalBytes);
|
||||
}
|
||||
bytesDoneBeforeCurrentFile += file.size;
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
console.error('Upload error:', error);
|
||||
showToast(`Failed to upload ${file.name}: ${error.message}`, 'error');
|
||||
failCount++;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
hideUploadProgress();
|
||||
}
|
||||
|
||||
if (successCount > 0) {
|
||||
showToast(`Uploaded ${successCount} file${successCount > 1 ? 's' : ''}${failCount > 0 ? ` (${failCount} failed)` : ''}`, 'success');
|
||||
loadObjects();
|
||||
}
|
||||
batchFiles.forEach(file => uploadQueue.push({ file, batch }));
|
||||
renderUploadProgress();
|
||||
drainUploadQueue();
|
||||
return completion;
|
||||
}
|
||||
|
||||
async function uploadMultipart(file, key, onProgress) {
|
||||
async function uploadMultipart(file, bucket, key, onProgress, onFinalizing, signal) {
|
||||
const contentType = file.type || 'application/octet-stream';
|
||||
|
||||
// Calculate optimal part size for this file
|
||||
const partSize = calculatePartSize(file.size);
|
||||
console.log(`Uploading ${file.name}: ${formatSize(file.size)}, part size: ${formatSize(partSize)}`);
|
||||
|
||||
// Initiate multipart upload
|
||||
const uploadId = await api.createMultipartUpload(currentBucket, key, contentType);
|
||||
|
||||
const parts = [];
|
||||
const totalParts = Math.ceil(file.size / partSize);
|
||||
let uploadId;
|
||||
|
||||
try {
|
||||
uploadId = await api.createMultipartUpload(bucket, key, contentType, signal);
|
||||
if (signal?.aborted) throw new DOMException('Upload cancelled', 'AbortError');
|
||||
for (let partNumber = 1; partNumber <= totalParts; partNumber++) {
|
||||
const start = (partNumber - 1) * partSize;
|
||||
const end = Math.min(start + partSize, file.size);
|
||||
const chunk = file.slice(start, end);
|
||||
const chunkBuffer = await chunk.arrayBuffer();
|
||||
|
||||
const etag = await api.uploadPart(currentBucket, key, uploadId, partNumber, chunkBuffer);
|
||||
const etag = await api.uploadPart(bucket, key, uploadId, partNumber, chunkBuffer, signal);
|
||||
parts.push({ partNumber, etag });
|
||||
|
||||
if (onProgress) onProgress(end);
|
||||
}
|
||||
|
||||
// Complete the multipart upload
|
||||
await api.completeMultipartUpload(currentBucket, key, uploadId, parts);
|
||||
if (signal?.aborted) throw new DOMException('Upload cancelled', 'AbortError');
|
||||
if (onFinalizing) onFinalizing();
|
||||
await api.completeMultipartUpload(bucket, key, uploadId, parts, signal);
|
||||
} catch (error) {
|
||||
// Abort the multipart upload on failure
|
||||
try {
|
||||
await api.abortMultipartUpload(currentBucket, key, uploadId);
|
||||
} catch (abortError) {
|
||||
console.error('Failed to abort multipart upload:', abortError);
|
||||
if (uploadId) {
|
||||
try {
|
||||
await api.abortMultipartUpload(bucket, key, uploadId);
|
||||
} catch (abortError) {
|
||||
console.error('Failed to abort multipart upload:', abortError);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
@@ -3407,7 +3561,7 @@ under the License.
|
||||
if (bytes === 0) return '0 B';
|
||||
const k = 1024;
|
||||
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length - 1);
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
|
||||
+8
-4
@@ -1395,7 +1395,7 @@ class VersityAPI {
|
||||
/**
|
||||
* Upload an object (PutObject) - for small files < 5MB
|
||||
*/
|
||||
async putObject(bucket, key, file, contentType = null) {
|
||||
async putObject(bucket, key, file, contentType = null, signal = undefined) {
|
||||
const finalContentType = contentType || file.type || 'application/octet-stream';
|
||||
const path = `/${bucket}/${encodeS3Key(key)}`;
|
||||
|
||||
@@ -1408,6 +1408,7 @@ class VersityAPI {
|
||||
method: 'PUT',
|
||||
headers: signed.headers,
|
||||
body: file,
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -1585,12 +1586,13 @@ class VersityAPI {
|
||||
* Initiate a multipart upload
|
||||
* Returns uploadId needed for subsequent parts
|
||||
*/
|
||||
async createMultipartUpload(bucket, key, contentType = 'application/octet-stream') {
|
||||
async createMultipartUpload(bucket, key, contentType = 'application/octet-stream', signal = undefined) {
|
||||
const fetchParams = await this.buildFetchParams('POST', `/${bucket}/${encodeS3Key(key)}`, { uploads: '' }, '', false, contentType);
|
||||
|
||||
const response = await fetch(fetchParams.url, {
|
||||
method: 'POST',
|
||||
headers: fetchParams.headers,
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -1614,7 +1616,7 @@ class VersityAPI {
|
||||
* Upload a single part of a multipart upload
|
||||
* Returns ETag needed for CompleteMultipartUpload
|
||||
*/
|
||||
async uploadPart(bucket, key, uploadId, partNumber, data) {
|
||||
async uploadPart(bucket, key, uploadId, partNumber, data, signal = undefined) {
|
||||
const arrayBuffer = data instanceof ArrayBuffer ? data : await data.arrayBuffer();
|
||||
const path = `/${bucket}/${encodeS3Key(key)}`;
|
||||
const queryParams = {
|
||||
@@ -1635,6 +1637,7 @@ class VersityAPI {
|
||||
method: 'PUT',
|
||||
headers: signed.headers,
|
||||
body: arrayBuffer,
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -1669,7 +1672,7 @@ class VersityAPI {
|
||||
* Complete a multipart upload
|
||||
* parts should be an array of { partNumber, etag }
|
||||
*/
|
||||
async completeMultipartUpload(bucket, key, uploadId, parts) {
|
||||
async completeMultipartUpload(bucket, key, uploadId, parts, signal = undefined) {
|
||||
let body = '<?xml version="1.0" encoding="UTF-8"?>\n<CompleteMultipartUpload>';
|
||||
|
||||
// Sort parts by partNumber
|
||||
@@ -1690,6 +1693,7 @@ class VersityAPI {
|
||||
method: 'POST',
|
||||
headers: fetchParams.headers,
|
||||
body: body,
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
||||
Reference in New Issue
Block a user