mirror of
https://github.com/versity/versitygw.git
synced 2026-09-25 01:14:22 +00:00
Merge pull request #2422 from versity/ben/web-explorer-concurrent-uploads
feat: add webui concurrent uploads with progress tracking
This commit is contained in:
Ben McClelland
+217
-63
@@ -2969,35 +2969,194 @@ under the License.
|
||||
// overall byte progress across every file/part of the current upload
|
||||
// batch, so a large file no longer looks stalled with no feedback.
|
||||
let uploadProgressEl = null;
|
||||
let nextUploadBatchId = 1;
|
||||
let activeUploadCount = 0;
|
||||
const uploadQueue = [];
|
||||
const uploadBatches = new Map();
|
||||
const MAX_CONCURRENT_UPLOADS = 3;
|
||||
|
||||
function getUploadStatusLabel(file) {
|
||||
if (file.state === 'completed') return 'Completed';
|
||||
if (file.state === 'failed') return 'Failed';
|
||||
if (file.state === 'cancelled') return 'Cancelled';
|
||||
if (file.state === 'cancelling') return 'Cancelling';
|
||||
if (file.state === 'finalizing') return 'Finalizing';
|
||||
if (file.state === 'uploading') return 'Uploading';
|
||||
return 'Queued';
|
||||
}
|
||||
|
||||
function renderUploadProgress() {
|
||||
if (uploadBatches.size === 0) {
|
||||
if (uploadProgressEl) {
|
||||
uploadProgressEl.remove();
|
||||
uploadProgressEl = null;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
function showUploadProgress(fileName, fileIndex, fileCount, bytesDone, bytesTotal) {
|
||||
if (!uploadProgressEl) {
|
||||
uploadProgressEl = document.createElement('div');
|
||||
uploadProgressEl.id = 'upload-progress-indicator';
|
||||
uploadProgressEl.className = 'fixed bottom-4 right-4 z-50 w-80 bg-white rounded-lg shadow-lg border border-gray-200 p-4';
|
||||
uploadProgressEl.className = 'fixed bottom-4 right-4 z-50 w-96 max-w-[calc(100vw-2rem)] bg-white rounded-lg shadow-lg border border-gray-200 p-4';
|
||||
uploadProgressEl.setAttribute('role', 'status');
|
||||
uploadProgressEl.setAttribute('aria-live', 'polite');
|
||||
const handleCancel = event => {
|
||||
const button = event.target.closest('[data-cancel-upload]');
|
||||
if (!button) return;
|
||||
if (event.type === 'pointerdown') event.preventDefault();
|
||||
cancelUpload(Number(button.dataset.batchId), Number(button.dataset.fileIndex));
|
||||
};
|
||||
uploadProgressEl.addEventListener('pointerdown', handleCancel);
|
||||
uploadProgressEl.addEventListener('click', handleCancel);
|
||||
document.body.appendChild(uploadProgressEl);
|
||||
}
|
||||
|
||||
const files = Array.from(uploadBatches.values()).flatMap(batch => batch.files);
|
||||
const trackedFiles = files.filter(file => file.state !== 'cancelled');
|
||||
const bytesDone = trackedFiles.reduce((sum, file) => sum + file.bytesDone, 0);
|
||||
const bytesTotal = trackedFiles.reduce((sum, file) => sum + file.size, 0);
|
||||
const percent = bytesTotal > 0 ? Math.min(100, Math.round((bytesDone / bytesTotal) * 100)) : 0;
|
||||
const label = fileCount > 1 ? `File ${fileIndex} of ${fileCount}: ${fileName}` : fileName;
|
||||
const completedCount = trackedFiles.filter(file => file.state === 'completed').length;
|
||||
const failedCount = trackedFiles.filter(file => file.state === 'failed').length;
|
||||
const cancelledCount = files.filter(file => file.state === 'cancelled').length;
|
||||
const activeCount = files.filter(file => file.state === 'uploading' || file.state === 'cancelling' || file.state === 'finalizing').length;
|
||||
const statusText = [
|
||||
`${completedCount} of ${trackedFiles.length} completed`,
|
||||
failedCount > 0 ? `${failedCount} failed` : null,
|
||||
cancelledCount > 0 ? `${cancelledCount} cancelled` : null
|
||||
].filter(Boolean).join(', ');
|
||||
|
||||
uploadProgressEl.innerHTML = `
|
||||
<div class="flex items-center justify-between mb-2 gap-2">
|
||||
<p class="text-sm font-medium text-gray-800 truncate" title="${escapeHtml(fileName)}">${escapeHtml(label)}</p>
|
||||
<span class="text-sm font-semibold text-blue-600 shrink-0">${percent}%</span>
|
||||
<p class="text-sm font-medium text-gray-800">Uploads${activeCount > 0 ? ` (${activeCount} active)` : ''}</p>
|
||||
<span class="text-sm font-semibold text-primary shrink-0">${percent}%</span>
|
||||
</div>
|
||||
<div class="w-full bg-gray-200 rounded-full h-2 overflow-hidden">
|
||||
<div class="bg-blue-600 h-2 rounded-full transition-all duration-150" style="width: ${percent}%"></div>
|
||||
<div class="w-full bg-gray-200 rounded-full h-2 overflow-hidden" role="progressbar" aria-valuemin="0" aria-valuemax="100" aria-valuenow="${percent}" aria-label="Upload progress">
|
||||
<div class="bg-primary h-2 rounded-full transition-all duration-150" style="width: ${percent}%"></div>
|
||||
</div>
|
||||
<p class="text-xs text-gray-500 mt-1">${escapeHtml(statusText)} · ${formatSize(bytesDone)} / ${formatSize(bytesTotal)}</p>
|
||||
<div class="mt-3 space-y-2 max-h-40 overflow-y-auto">
|
||||
${files.filter(file => file.state !== 'completed' && file.state !== 'cancelled').map(file => {
|
||||
const filePercent = file.size > 0 ? Math.min(100, Math.round((file.bytesDone / file.size) * 100)) : 0;
|
||||
const statusLabel = getUploadStatusLabel(file);
|
||||
return `
|
||||
<div class="text-xs" aria-label="${escapeHtml(file.name)}: ${statusLabel}">
|
||||
<div class="flex items-center justify-between gap-2">
|
||||
<span class="truncate text-gray-700" title="${escapeHtml(file.name)}">${escapeHtml(file.name)}</span>
|
||||
<div class="flex items-center gap-2 shrink-0">
|
||||
<span class="${file.state === 'failed' ? 'text-red-600' : 'text-gray-500'}">${statusLabel}</span>
|
||||
${file.state === 'queued' || file.state === 'uploading' || file.state === 'finalizing' ? `<button type="button" data-cancel-upload data-batch-id="${file.batchId}" data-file-index="${file.fileIndex}" class="text-red-600 hover:text-red-700 font-medium" aria-label="Cancel ${escapeHtml(file.name)}">Cancel</button>` : ''}
|
||||
</div>
|
||||
</div>
|
||||
<div class="w-full bg-gray-100 rounded-full h-1 mt-1 overflow-hidden">
|
||||
<div class="${file.state === 'failed' ? 'bg-red-400' : 'bg-primary'} h-1 rounded-full" style="width: ${filePercent}%"></div>
|
||||
</div>
|
||||
</div>
|
||||
`;
|
||||
}).join('')}
|
||||
</div>
|
||||
<p class="text-xs text-gray-500 mt-1">${formatSize(bytesDone)} / ${formatSize(bytesTotal)}</p>
|
||||
`;
|
||||
|
||||
}
|
||||
|
||||
function hideUploadProgress() {
|
||||
if (uploadProgressEl) {
|
||||
uploadProgressEl.remove();
|
||||
uploadProgressEl = null;
|
||||
function removeFinishedUploadBatches() {
|
||||
if (uploadQueue.length > 0 || activeUploadCount > 0) return;
|
||||
uploadBatches.clear();
|
||||
renderUploadProgress();
|
||||
}
|
||||
|
||||
function updateUploadFileProgress(file, bytesDone) {
|
||||
file.bytesDone = Math.max(file.bytesDone, Math.min(bytesDone, file.size));
|
||||
renderUploadProgress();
|
||||
}
|
||||
|
||||
function finishUploadBatch(batch) {
|
||||
if (batch.pendingFiles > 0) return;
|
||||
|
||||
const successfulFiles = batch.files.filter(file => file.state === 'completed').length;
|
||||
const failedFiles = batch.files.filter(file => file.state === 'failed').length;
|
||||
if (successfulFiles > 0) {
|
||||
showToast(`Uploaded ${successfulFiles} file${successfulFiles > 1 ? 's' : ''}${failedFiles > 0 ? ` (${failedFiles} failed)` : ''}`, 'success');
|
||||
if (currentBucket === batch.bucket && currentPrefix === batch.prefix) {
|
||||
loadObjects();
|
||||
}
|
||||
}
|
||||
batch.resolve();
|
||||
removeFinishedUploadBatches();
|
||||
}
|
||||
|
||||
function cancelUpload(batchId, fileIndex) {
|
||||
const batch = uploadBatches.get(batchId);
|
||||
const file = batch?.files[fileIndex];
|
||||
if (!file || (file.state !== 'queued' && file.state !== 'uploading' && file.state !== 'finalizing')) return;
|
||||
|
||||
if (file.state === 'queued') {
|
||||
const queueIndex = uploadQueue.findIndex(item => item.file === file);
|
||||
if (queueIndex !== -1) uploadQueue.splice(queueIndex, 1);
|
||||
file.state = 'cancelled';
|
||||
batch.pendingFiles--;
|
||||
showToast(`Cancelled upload: ${file.name}`, 'info');
|
||||
renderUploadProgress();
|
||||
finishUploadBatch(batch);
|
||||
drainUploadQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
file.state = 'cancelling';
|
||||
file.controller?.abort();
|
||||
renderUploadProgress();
|
||||
}
|
||||
|
||||
async function runUploadFile(file, batch) {
|
||||
file.state = 'uploading';
|
||||
file.controller = new AbortController();
|
||||
renderUploadProgress();
|
||||
|
||||
try {
|
||||
if (file.size >= MULTIPART_THRESHOLD) {
|
||||
await uploadMultipart(file.file, batch.bucket, file.key, (bytesDone) => {
|
||||
updateUploadFileProgress(file, bytesDone);
|
||||
}, () => {
|
||||
file.state = 'finalizing';
|
||||
renderUploadProgress();
|
||||
}, file.controller.signal);
|
||||
} else {
|
||||
await api.putObject(batch.bucket, file.key, file.file, null, file.controller.signal);
|
||||
updateUploadFileProgress(file, file.size);
|
||||
}
|
||||
if (file.state === 'cancelling') {
|
||||
file.state = 'cancelled';
|
||||
} else {
|
||||
file.bytesDone = file.size;
|
||||
file.state = 'completed';
|
||||
}
|
||||
} catch (error) {
|
||||
if (file.state === 'cancelling' || error.name === 'AbortError') {
|
||||
file.state = 'cancelled';
|
||||
showToast(`Cancelled upload: ${file.name}`, 'info');
|
||||
} else {
|
||||
file.state = 'failed';
|
||||
file.error = error;
|
||||
console.error('Upload error:', error);
|
||||
showToast(`Failed to upload ${file.name}: ${error.message}`, 'error');
|
||||
}
|
||||
} finally {
|
||||
file.controller = null;
|
||||
batch.pendingFiles--;
|
||||
activeUploadCount--;
|
||||
renderUploadProgress();
|
||||
finishUploadBatch(batch);
|
||||
drainUploadQueue();
|
||||
}
|
||||
}
|
||||
|
||||
function drainUploadQueue() {
|
||||
while (activeUploadCount < MAX_CONCURRENT_UPLOADS && uploadQueue.length > 0) {
|
||||
const item = uploadQueue.shift();
|
||||
activeUploadCount++;
|
||||
runUploadFile(item.file, item.batch);
|
||||
}
|
||||
removeFinishedUploadBatches();
|
||||
}
|
||||
|
||||
function openUploadDialog() {
|
||||
@@ -3036,88 +3195,83 @@ under the License.
|
||||
}
|
||||
|
||||
async function uploadFiles(files) {
|
||||
let successCount = 0;
|
||||
let failCount = 0;
|
||||
const uploadBucket = currentBucket;
|
||||
const uploadPrefix = currentPrefix;
|
||||
const batchId = nextUploadBatchId++;
|
||||
const batchFiles = files.map((file, index) => ({
|
||||
file,
|
||||
batchId,
|
||||
fileIndex: index,
|
||||
name: file.name,
|
||||
key: uploadPrefix + file.name,
|
||||
size: file.size,
|
||||
bytesDone: 0,
|
||||
state: 'queued',
|
||||
error: null,
|
||||
controller: null
|
||||
}));
|
||||
const batch = {
|
||||
id: batchId,
|
||||
bucket: uploadBucket,
|
||||
prefix: uploadPrefix,
|
||||
files: batchFiles,
|
||||
pendingFiles: batchFiles.length,
|
||||
resolve: null
|
||||
};
|
||||
const completion = new Promise(resolve => {
|
||||
batch.resolve = resolve;
|
||||
});
|
||||
uploadBatches.set(batchId, batch);
|
||||
|
||||
// Show uploading toast for multiple files
|
||||
if (files.length > 1) {
|
||||
showToast(`Uploading ${files.length} files...`, 'info');
|
||||
}
|
||||
|
||||
const totalBytes = files.reduce((sum, f) => sum + f.size, 0);
|
||||
let bytesDoneBeforeCurrentFile = 0;
|
||||
|
||||
try {
|
||||
for (let i = 0; i < files.length; i++) {
|
||||
const file = files[i];
|
||||
const key = currentPrefix + file.name;
|
||||
|
||||
try {
|
||||
if (file.size >= MULTIPART_THRESHOLD) {
|
||||
// Use multipart upload for large files
|
||||
await uploadMultipart(file, key, (bytesDoneInFile) => {
|
||||
showUploadProgress(file.name, i + 1, files.length, bytesDoneBeforeCurrentFile + bytesDoneInFile, totalBytes);
|
||||
});
|
||||
} else {
|
||||
// Use simple PUT for small files (no per-chunk hook to report progress
|
||||
// through, so report it in one step once the PUT actually resolves —
|
||||
// matches the multipart path only firing onProgress after each part is
|
||||
// confirmed uploaded, never optimistically before)
|
||||
await api.putObject(currentBucket, key, file);
|
||||
showUploadProgress(file.name, i + 1, files.length, bytesDoneBeforeCurrentFile + file.size, totalBytes);
|
||||
}
|
||||
bytesDoneBeforeCurrentFile += file.size;
|
||||
successCount++;
|
||||
} catch (error) {
|
||||
console.error('Upload error:', error);
|
||||
showToast(`Failed to upload ${file.name}: ${error.message}`, 'error');
|
||||
failCount++;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
hideUploadProgress();
|
||||
}
|
||||
|
||||
if (successCount > 0) {
|
||||
showToast(`Uploaded ${successCount} file${successCount > 1 ? 's' : ''}${failCount > 0 ? ` (${failCount} failed)` : ''}`, 'success');
|
||||
loadObjects();
|
||||
}
|
||||
batchFiles.forEach(file => uploadQueue.push({ file, batch }));
|
||||
renderUploadProgress();
|
||||
drainUploadQueue();
|
||||
return completion;
|
||||
}
|
||||
|
||||
async function uploadMultipart(file, key, onProgress) {
|
||||
async function uploadMultipart(file, bucket, key, onProgress, onFinalizing, signal) {
|
||||
const contentType = file.type || 'application/octet-stream';
|
||||
|
||||
// Calculate optimal part size for this file
|
||||
const partSize = calculatePartSize(file.size);
|
||||
console.log(`Uploading ${file.name}: ${formatSize(file.size)}, part size: ${formatSize(partSize)}`);
|
||||
|
||||
// Initiate multipart upload
|
||||
const uploadId = await api.createMultipartUpload(currentBucket, key, contentType);
|
||||
|
||||
const parts = [];
|
||||
const totalParts = Math.ceil(file.size / partSize);
|
||||
let uploadId;
|
||||
|
||||
try {
|
||||
uploadId = await api.createMultipartUpload(bucket, key, contentType, signal);
|
||||
if (signal?.aborted) throw new DOMException('Upload cancelled', 'AbortError');
|
||||
for (let partNumber = 1; partNumber <= totalParts; partNumber++) {
|
||||
const start = (partNumber - 1) * partSize;
|
||||
const end = Math.min(start + partSize, file.size);
|
||||
const chunk = file.slice(start, end);
|
||||
const chunkBuffer = await chunk.arrayBuffer();
|
||||
|
||||
const etag = await api.uploadPart(currentBucket, key, uploadId, partNumber, chunkBuffer);
|
||||
const etag = await api.uploadPart(bucket, key, uploadId, partNumber, chunkBuffer, signal);
|
||||
parts.push({ partNumber, etag });
|
||||
|
||||
if (onProgress) onProgress(end);
|
||||
}
|
||||
|
||||
// Complete the multipart upload
|
||||
await api.completeMultipartUpload(currentBucket, key, uploadId, parts);
|
||||
if (signal?.aborted) throw new DOMException('Upload cancelled', 'AbortError');
|
||||
if (onFinalizing) onFinalizing();
|
||||
await api.completeMultipartUpload(bucket, key, uploadId, parts, signal);
|
||||
} catch (error) {
|
||||
// Abort the multipart upload on failure
|
||||
try {
|
||||
await api.abortMultipartUpload(currentBucket, key, uploadId);
|
||||
} catch (abortError) {
|
||||
console.error('Failed to abort multipart upload:', abortError);
|
||||
if (uploadId) {
|
||||
try {
|
||||
await api.abortMultipartUpload(bucket, key, uploadId);
|
||||
} catch (abortError) {
|
||||
console.error('Failed to abort multipart upload:', abortError);
|
||||
}
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
@@ -3407,7 +3561,7 @@ under the License.
|
||||
if (bytes === 0) return '0 B';
|
||||
const k = 1024;
|
||||
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
const i = Math.min(Math.floor(Math.log(bytes) / Math.log(k)), sizes.length - 1);
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
|
||||
+8
-4
@@ -1395,7 +1395,7 @@ class VersityAPI {
|
||||
/**
|
||||
* Upload an object (PutObject) - for small files < 5MB
|
||||
*/
|
||||
async putObject(bucket, key, file, contentType = null) {
|
||||
async putObject(bucket, key, file, contentType = null, signal = undefined) {
|
||||
const finalContentType = contentType || file.type || 'application/octet-stream';
|
||||
const path = `/${bucket}/${encodeS3Key(key)}`;
|
||||
|
||||
@@ -1408,6 +1408,7 @@ class VersityAPI {
|
||||
method: 'PUT',
|
||||
headers: signed.headers,
|
||||
body: file,
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -1585,12 +1586,13 @@ class VersityAPI {
|
||||
* Initiate a multipart upload
|
||||
* Returns uploadId needed for subsequent parts
|
||||
*/
|
||||
async createMultipartUpload(bucket, key, contentType = 'application/octet-stream') {
|
||||
async createMultipartUpload(bucket, key, contentType = 'application/octet-stream', signal = undefined) {
|
||||
const fetchParams = await this.buildFetchParams('POST', `/${bucket}/${encodeS3Key(key)}`, { uploads: '' }, '', false, contentType);
|
||||
|
||||
const response = await fetch(fetchParams.url, {
|
||||
method: 'POST',
|
||||
headers: fetchParams.headers,
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -1614,7 +1616,7 @@ class VersityAPI {
|
||||
* Upload a single part of a multipart upload
|
||||
* Returns ETag needed for CompleteMultipartUpload
|
||||
*/
|
||||
async uploadPart(bucket, key, uploadId, partNumber, data) {
|
||||
async uploadPart(bucket, key, uploadId, partNumber, data, signal = undefined) {
|
||||
const arrayBuffer = data instanceof ArrayBuffer ? data : await data.arrayBuffer();
|
||||
const path = `/${bucket}/${encodeS3Key(key)}`;
|
||||
const queryParams = {
|
||||
@@ -1635,6 +1637,7 @@ class VersityAPI {
|
||||
method: 'PUT',
|
||||
headers: signed.headers,
|
||||
body: arrayBuffer,
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
@@ -1669,7 +1672,7 @@ class VersityAPI {
|
||||
* Complete a multipart upload
|
||||
* parts should be an array of { partNumber, etag }
|
||||
*/
|
||||
async completeMultipartUpload(bucket, key, uploadId, parts) {
|
||||
async completeMultipartUpload(bucket, key, uploadId, parts, signal = undefined) {
|
||||
let body = '<?xml version="1.0" encoding="UTF-8"?>\n<CompleteMultipartUpload>';
|
||||
|
||||
// Sort parts by partNumber
|
||||
@@ -1690,6 +1693,7 @@ class VersityAPI {
|
||||
method: 'POST',
|
||||
headers: fetchParams.headers,
|
||||
body: body,
|
||||
signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
|
||||
Reference in New Issue
Block a user