mirror of
https://github.com/scylladb/scylladb.git
synced 2026-05-22 15:52:13 +00:00
9faaf1f09c
Move S3/GCS server classes (S3Server, MinioWrapper, GSFront, GSServer), factory functions (create_s3_server, create_gs_server), CQL helpers (format_tuples, keyspace_options), bucket naming (_make_bucket_name), and the s3_server fixture from test/cluster/object_store/conftest.py into a shared module at test/pylib/object_storage.py. The conftest.py is now a thin wrapper that re-exports symbols and defines only the fixtures specific to the object_store suite (object_storage, s3_storage). All external importers are updated. Old class names (S3_Server, GSServer) are kept as aliases for backward compatibility.
341 lines
12 KiB
Python
341 lines
12 KiB
Python
#
|
|
# Copyright (C) 2023-present ScyllaDB
|
|
#
|
|
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.1
|
|
#
|
|
|
|
"""Reusable object-storage helpers for S3 and GCS test fixtures.
|
|
|
|
Classes, factory functions, and shared pytest fixtures used by
|
|
test/cluster/object_store/ and test/cqlpy/ tests.
|
|
"""
|
|
|
|
import os
|
|
import logging
|
|
import re
|
|
import uuid
|
|
|
|
from test.pylib.minio_server import MinioServer
|
|
from test.pylib.dockerized_service import DockerizedServer
|
|
from operator import attrgetter
|
|
|
|
import pytest
|
|
import boto3
|
|
import requests
|
|
|
|
|
|
def format_tuples(tuples=None, **kwargs):
|
|
'''format a dict to structured values (tuples) in CQL'''
|
|
if tuples is None:
|
|
tuples = {}
|
|
tuples.update(kwargs)
|
|
body = ', '.join(f"'{key}': '{value}'" for key, value in tuples.items())
|
|
return f'{{ {body} }}'
|
|
|
|
|
|
def keyspace_options(object_storage, rf=1):
|
|
storage_opts = format_tuples(type=f'{object_storage.type}', endpoint=object_storage.address, bucket=object_storage.bucket_name)
|
|
return f"WITH replication = {{'class': 'NetworkTopologyStrategy', 'replication_factor': {rf}}} AND STORAGE = {storage_opts}"
|
|
|
|
|
|
def _make_bucket_name(test_name: str) -> str:
|
|
"""Generate a valid S3 bucket name from a pytest test name.
|
|
|
|
S3 bucket naming rules: lowercase, digits, hyphens only; 3-63 chars;
|
|
must start/end with a letter or digit.
|
|
"""
|
|
# Lowercase, replace non-alphanumeric runs with a single hyphen, strip leading/trailing hyphens
|
|
name = re.sub(r'[^a-z0-9]+', '-', test_name.lower()).strip('-')
|
|
# Add a short unique suffix to avoid collisions (e.g. parametrized tests with same prefix)
|
|
suffix = uuid.uuid4().hex[:8]
|
|
# Truncate so total length (name + '-' + suffix) <= 63
|
|
max_prefix = 63 - len(suffix) - 1
|
|
name = name[:max_prefix].rstrip('-')
|
|
return f"{name}-{suffix}"
|
|
|
|
|
|
class S3Server:
|
|
def __init__(self, tempdir: str, address: str, port: int, acc_key: str, secret_key: str, region: str, bucket_name):
|
|
self.tempdir = tempdir
|
|
self.ip = address
|
|
self.port = port
|
|
self.address = f'http://{self.ip}:{self.port}'
|
|
self.acc_key = acc_key
|
|
self.secret_key = secret_key
|
|
self.region = region
|
|
self.bucket_name = bucket_name
|
|
|
|
def __repr__(self):
|
|
return f"[unknown] {self.address}/{self.bucket_name}"
|
|
|
|
@property
|
|
def type(self):
|
|
return 'S3'
|
|
|
|
def create_endpoint_conf(self):
|
|
return MinioServer.create_conf(self.address, self.region)
|
|
|
|
def get_resource(self):
|
|
"""Creates boto3.resource object that can be used to communicate to the given server"""
|
|
return boto3.resource('s3',
|
|
endpoint_url=self.address,
|
|
aws_access_key_id=self.acc_key,
|
|
aws_secret_access_key=self.secret_key,
|
|
aws_session_token=None,
|
|
config=boto3.session.Config(signature_version='s3v4'),
|
|
verify=False
|
|
)
|
|
|
|
def create_test_bucket(self, test_name: str):
|
|
"""Create a unique per-test bucket using boto3."""
|
|
self.bucket_name = _make_bucket_name(test_name)
|
|
resource = self.get_resource()
|
|
resource.Bucket(self.bucket_name).create()
|
|
|
|
def destroy_test_bucket(self):
|
|
"""Empty and delete the per-test bucket using boto3."""
|
|
try:
|
|
resource = self.get_resource()
|
|
bucket = resource.Bucket(self.bucket_name)
|
|
bucket.objects.all().delete()
|
|
bucket.delete()
|
|
except Exception as e:
|
|
logging.warning("Failed to destroy test bucket %s: %s", self.bucket_name, e)
|
|
|
|
async def start(self):
|
|
pass
|
|
|
|
async def stop(self):
|
|
pass
|
|
|
|
|
|
# Keep the old name as an alias for backward compatibility
|
|
S3_Server = S3Server
|
|
|
|
|
|
class MinioWrapper(S3Server):
|
|
def __init__(self, tempdir):
|
|
self.server = MinioServer(tempdir,
|
|
'127.0.0.1',
|
|
logging.getLogger('minio'))
|
|
super().__init__(
|
|
tempdir,
|
|
self.server.address,
|
|
self.server.port,
|
|
self.server.access_key,
|
|
self.server.secret_key,
|
|
MinioServer.DEFAULT_REGION,
|
|
self.server.bucket_name)
|
|
|
|
def create_endpoint_conf(self):
|
|
return MinioServer.create_conf(self.address, self.region)
|
|
|
|
async def start(self):
|
|
return self.server.start()
|
|
|
|
async def stop(self):
|
|
return self.server.stop()
|
|
|
|
|
|
class GSFront:
|
|
def __init__(self, endpoint, bucket_name, credentials_file):
|
|
self.endpoint = endpoint
|
|
self.bucket_name = bucket_name
|
|
self.credentials_file = credentials_file
|
|
|
|
@property
|
|
def address(self):
|
|
return self.endpoint
|
|
|
|
@property
|
|
def type(self):
|
|
return 'GS'
|
|
|
|
def create_endpoint_conf(self):
|
|
endpoint = {'name': self.endpoint,
|
|
'type': 'gs',
|
|
'credentials_file': self.credentials_file if self.credentials_file else 'none'
|
|
}
|
|
return [endpoint]
|
|
|
|
def get_resource(self):
|
|
"""Creates boto3.resource object that can be used to communicate to the given server"""
|
|
return boto3.resource('s3',
|
|
endpoint_url=self.endpoint,
|
|
config=boto3.session.Config(signature_version='s3v4'),
|
|
verify=False
|
|
)
|
|
|
|
def create_test_bucket(self, test_name: str):
|
|
"""Create a unique per-test bucket using boto3."""
|
|
self.bucket_name = _make_bucket_name(test_name)
|
|
resource = self.get_resource()
|
|
resource.Bucket(self.bucket_name).create()
|
|
|
|
def destroy_test_bucket(self):
|
|
"""Empty and delete the per-test bucket using boto3."""
|
|
try:
|
|
resource = self.get_resource()
|
|
bucket = resource.Bucket(self.bucket_name)
|
|
bucket.objects.all().delete()
|
|
bucket.delete()
|
|
except Exception as e:
|
|
logging.warning("Failed to destroy test bucket %s: %s", self.bucket_name, e)
|
|
|
|
async def start(self):
|
|
pass
|
|
|
|
async def stop(self):
|
|
pass
|
|
|
|
|
|
class GSServerImpl(GSFront):
|
|
def __init__(self, tmpdir):
|
|
super(GSServerImpl, self).__init__(None, 'testbucket', None)
|
|
self.server = None
|
|
self.host = None
|
|
self.port = None
|
|
self.oldvars = {}
|
|
self.vars = { 'GS_SERVER_ADDRESS_FOR_TEST': attrgetter('endpoint'),
|
|
'GS_BUCKET_FOR_TEST': attrgetter('bucket_name'),
|
|
'GS_CREDENTIALS_FILE': attrgetter('credentials_file'),
|
|
}
|
|
self.tmpdir = tmpdir
|
|
|
|
def publish(self):
|
|
self.endpoint = f'http://{self.host}:{self.port}'
|
|
self.oldvars = { k : os.environ.get(k) for k in self.vars }
|
|
for k, v in self.vars.items():
|
|
val = v(self)
|
|
if val:
|
|
os.environ[k] = val
|
|
|
|
def unpublish(self):
|
|
for k in self.vars:
|
|
v = self.oldvars.get(k)
|
|
if v:
|
|
os.environ[k] = v
|
|
elif os.environ.get(k):
|
|
del os.environ[k]
|
|
|
|
def _image_args(self, host, port):
|
|
# pylint: disable=unused-argument
|
|
# note: need to set 'public-host' to the IP we connect to, to make XML (s3) API work for listing
|
|
return ["-scheme", "http", "-log-level", "debug", "--port", f'{port}', '-public-host', '127.0.0.1']
|
|
|
|
async def start(self):
|
|
self.server = DockerizedServer("docker.io/fsouza/fake-gcs-server:1.52.3", self.tmpdir,
|
|
logfilenamebase="fake-gcs-server",
|
|
image_args=self._image_args,
|
|
success_string="server started at",
|
|
failure_string="address already in use",
|
|
port=4443
|
|
)
|
|
await self.server.start()
|
|
self.port = self.server.port
|
|
self.host = self.server.host
|
|
self.publish()
|
|
|
|
|
|
def create_test_bucket(self, test_name: str):
|
|
"""Create a unique per-test bucket using GCS HTTP API (fake server doesn't support S3 XML for this)."""
|
|
self.bucket_name = _make_bucket_name(test_name)
|
|
response = requests.post(f'{self.endpoint}/storage/v1/b?project=testproject', json={
|
|
'name': self.bucket_name, 'location': 'US', 'storageClass': 'STANDARD',
|
|
'iamConfiguration': {
|
|
'uniformBucketLevelAccess': {
|
|
'enabled': True,
|
|
}
|
|
}
|
|
}, timeout=10)
|
|
if response.status_code not in [200, 201]:
|
|
raise Exception(f'Could not create test bucket: {response}')
|
|
|
|
def destroy_test_bucket(self):
|
|
"""Empty and delete the per-test bucket using GCS HTTP API."""
|
|
try:
|
|
# List and delete all objects first using boto3 (listing works on fake GCS)
|
|
resource = self.get_resource()
|
|
bucket = resource.Bucket(self.bucket_name)
|
|
bucket.objects.all().delete()
|
|
# Delete the bucket via GCS HTTP API
|
|
requests.delete(f'{self.endpoint}/storage/v1/b/{self.bucket_name}', timeout=10)
|
|
except Exception as e:
|
|
logging.warning("Failed to destroy test bucket %s: %s", self.bucket_name, e)
|
|
|
|
async def stop(self):
|
|
if self.server:
|
|
await self.server.stop()
|
|
self.unpublish()
|
|
|
|
|
|
# Keep the old name as an alias for backward compatibility (conftest.py imports it)
|
|
GSServer = GSServerImpl
|
|
|
|
|
|
def create_s3_server(pytestconfig, tmpdir):
|
|
server = None
|
|
s3_server_address = pytestconfig.getoption('--s3-server-address')
|
|
s3_server_port = pytestconfig.getoption('--s3-server-port')
|
|
aws_acc_key = pytestconfig.getoption('--aws-access-key')
|
|
aws_secret_key = pytestconfig.getoption('--aws-secret-key')
|
|
aws_region = pytestconfig.getoption('--aws-region')
|
|
s3_server_bucket = pytestconfig.getoption('--s3-server-bucket')
|
|
|
|
default_address = os.environ.get(MinioServer.ENV_ADDRESS)
|
|
default_port = os.environ.get(MinioServer.ENV_PORT)
|
|
default_acc_key = os.environ.get(MinioServer.ENV_ACCESS_KEY)
|
|
default_secret_key = os.environ.get(MinioServer.ENV_SECRET_KEY)
|
|
default_region = MinioServer.DEFAULT_REGION
|
|
default_bucket = os.environ.get(MinioServer.ENV_BUCKET)
|
|
|
|
# Note: bucket_name passed here is overwritten by create_test_bucket() when
|
|
# using the s3_server fixture. The --s3-server-bucket option currently only
|
|
# affects code paths that use server.bucket_name directly without the fixture.
|
|
tempdir = tmpdir.strpath
|
|
if s3_server_address:
|
|
server = S3Server(tempdir,
|
|
s3_server_address,
|
|
s3_server_port,
|
|
aws_acc_key,
|
|
aws_secret_key,
|
|
aws_region,
|
|
s3_server_bucket)
|
|
elif default_address:
|
|
server = S3Server(tempdir,
|
|
default_address,
|
|
int(default_port),
|
|
default_acc_key,
|
|
default_secret_key,
|
|
default_region,
|
|
default_bucket)
|
|
else:
|
|
server = MinioWrapper(tempdir)
|
|
return server
|
|
|
|
|
|
def create_gs_server(tmpdir):
|
|
endpoint = os.environ.get('GS_SERVER_ADDRESS_FOR_TEST')
|
|
bucket = os.environ.get('GS_BUCKET_FOR_TEST')
|
|
credentials_file = os.environ.get('GS_CREDENTIALS_FILE')
|
|
|
|
if endpoint is not None and bucket is not None:
|
|
return GSFront(endpoint, bucket, credentials_file)
|
|
return GSServerImpl(tmpdir)
|
|
|
|
|
|
@pytest.fixture(scope="function")
|
|
async def s3_server(request, pytestconfig, tmpdir):
|
|
server = create_s3_server(pytestconfig, tmpdir)
|
|
await server.start()
|
|
bucket_created = False
|
|
try:
|
|
server.create_test_bucket(request.node.name)
|
|
bucket_created = True
|
|
yield server
|
|
finally:
|
|
if bucket_created:
|
|
server.destroy_test_bucket()
|
|
await server.stop()
|
|
|