From b58ad040d2abd7bc307b9ecb0ddb08adf2d4ee5e Mon Sep 17 00:00:00 2001 From: Pavel Emelyanov Date: Wed, 19 Apr 2023 15:00:30 +0300 Subject: [PATCH] sstables: Switch data and index sink to use jumbo uploader These two can grow large. Non-jumbo sink is effectively limited with 10000 parts, since each is ~5Mb the maximum uploadable data/index happens to be 50Gb which is too small. Other components shouldn't grow that big and continue using simple and a bit faster uploading sink. Signed-off-by: Pavel Emelyanov --- sstables/storage.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sstables/storage.cc b/sstables/storage.cc index 61d4c9f157..42c5c36c1e 100644 --- a/sstables/storage.cc +++ b/sstables/storage.cc @@ -494,7 +494,8 @@ future s3_storage::open_component(const sstable& sst, component_type type, future s3_storage::make_data_or_index_sink(sstable& sst, component_type type, io_priority_class pc) { assert(type == component_type::Data || type == component_type::Index); co_await ensure_remote_prefix(sst); - co_return _client->make_upload_sink(make_s3_object_name(sst, type)); + // FIXME: if we have file size upper bound upfront, it's better to use make_upload_sink() instead + co_return _client->make_upload_jumbo_sink(make_s3_object_name(sst, type)); } future s3_storage::make_component_sink(sstable& sst, component_type type, open_flags oflags, file_output_stream_options options) {