mirrors/scylladb
1
0
Fork 0
mirror of https://github.com/scylladb/scylladb.git synced 2026-05-29 19:21:01 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'Refactor and enhance s3_tests' from Ernest Zaslavsky

Browse Source 
This PR introduces a cleanup mechanism in s3_tests to remove uploaded objects after the test completes, ensuring a clean testing environment. Additionally, the recently added test has been refactored and split into smaller, more maintainable parts, improving readability and extending its coverage to include the "proxied" case.

As these changes primarily improve code aesthetics and maintainability, backporting is not necessary.

Refs: https://github.com/scylladb/scylladb/issues/23830

Closes scylladb/scylladb#23828

* github.com:scylladb/scylladb:
  s3_tests: Improve and extend copy object test coverage
  s3_tests: Implement post-test cleanup for uploaded objects
This commit is contained in:
Pavel Emelyanov
2025-04-22 16:40:37 +03:00
parent 5fd2eabd48 edaa3f4bdd
commit 65efd2b2f6
1 changed files with 27 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
test/boost/s3_test.cc
View File

@@ -294,6 +294,7 @@ future<> test_client_upload_file(const client_maker_function& client_maker, std:
co_await readable_file.close();
co_await input.close();
co_await client->delete_object(object_name);
co_await client->close();
}
@@ -584,6 +585,7 @@ SEASTAR_THREAD_TEST_CASE(test_object_reupload) {
semaphore mem(16 << 20);
auto cln = make_minio_client(mem);
auto close_client = deferred_close(*cln);
auto delete_object = deferred_delete_object(cln, name);
constexpr std::string_view content{"1234567890"};
for (auto i : {1, 2}) {
testlog.info("Put object {}, iteration {}", name, i);
@@ -633,6 +635,7 @@ void test_download_data_source(const client_maker_function& client_maker, unsign
semaphore mem(16<<20);
auto cln = client_maker(mem);
auto close_client = deferred_close(*cln);
auto delete_object = deferred_delete_object(cln, name);
static constexpr unsigned chunk_size = 1000;
testlog.info("Preparation: Upload object");
@@ -664,33 +667,20 @@ SEASTAR_THREAD_TEST_CASE(test_download_data_source_proxy) {
test_download_data_source(make_proxy_client, 3 * 1024);
}
SEASTAR_THREAD_TEST_CASE(test_object_copy) {
void test_object_copy(const client_maker_function& client_maker, size_t chunk_size, size_t chunks) {
const sstring name(fmt::format("/{}/testobject-{}", tests::getenv_safe("S3_BUCKET_FOR_TEST"), ::getpid()));
const sstring name_copy(fmt::format("/{}/testobject-{}-copy", tests::getenv_safe("S3_BUCKET_FOR_TEST"), ::getpid()));
semaphore mem(16 << 20);
auto cln = make_minio_client(mem);
auto cln = client_maker(mem);;
auto close_client = deferred_close(*cln);
constexpr std::string_view content{"1234567890"};
testlog.info("Put object {}", name);
temporary_buffer<char> data = sstring(content).release();
cln->put_object(name, std::move(data)).get();
testlog.info("Copy object {}", name);
cln->copy_object(name, name_copy).get();
size_t sz = cln->get_object_size(name_copy).get();
BOOST_REQUIRE_EQUAL(sz, content.size());
auto delete_object = deferred_delete_object(cln, name);
auto delete_copy_object = deferred_delete_object(cln, name_copy);
auto out = output_stream<char>(cln->make_upload_sink(name));
constexpr size_t chunk_size = 1_MiB;
constexpr size_t writes = 11;
auto rnd = tests::random::get_bytes(chunk_size);
for (unsigned ch = 0; ch < writes; ch++) {
for (unsigned ch = 0; ch < chunks; ch++) {
out.write(reinterpret_cast<char*>(rnd.begin()), rnd.size()).get();
}
@@ -698,8 +688,8 @@ SEASTAR_THREAD_TEST_CASE(test_object_copy) {
out.close().get();
cln->copy_object(name, name_copy, 5_MiB).get();
sz = cln->get_object_size(name_copy).get();
BOOST_REQUIRE_EQUAL(sz, chunk_size * writes);
auto sz = cln->get_object_size(name_copy).get();
BOOST_REQUIRE_EQUAL(sz, chunk_size * chunks);
for (size_t off = 0; off < sz; off += chunk_size) {
auto len = std::min(chunk_size, sz - off);
@@ -711,6 +701,22 @@ SEASTAR_THREAD_TEST_CASE(test_object_copy) {
}
}
SEASTAR_THREAD_TEST_CASE(test_small_object_copy) {
test_object_copy(make_minio_client, 1000, 2);
}
SEASTAR_THREAD_TEST_CASE(test_large_object_copy) {
test_object_copy(make_minio_client, 1_MiB, 6);
}
SEASTAR_THREAD_TEST_CASE(test_small_object_copy_proxy) {
test_object_copy(make_proxy_client, 1000, 2);
}
SEASTAR_THREAD_TEST_CASE(test_large_object_copy_proxy) {
test_object_copy(make_proxy_client, 1_MiB, 6);
}
SEASTAR_THREAD_TEST_CASE(test_creds) {
/*
* Note: This test does not cover the functionality of the `aws::environment_aws_credentials_provider` class because proper testing requires altering