4e87f95b42
The test test_table.py::test_table_streams_on creates tables with various stream types, and then immediately deletes them without testing anything. This is a slow test (taking almost a full second on my laptop), and is redundant because in test_streams.py we have tests which create tables with streams in the same way - but then actually test that things work with these streams. So this test might as well be removed, and this is what we do in this patch. Removing this test shaves another second from the Alternator test suite's run time. Signed-off-by: Nadav Har'El <nyh@scylladb.com> Message-Id: <20210317230530.1780849-1-nyh@scylladb.com>
320 lines
15 KiB
Python
320 lines
15 KiB
Python
# Copyright 2019 ScyllaDB
|
|
#
|
|
# This file is part of Scylla.
|
|
#
|
|
# Scylla is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Scylla is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
# Tests for basic table operations: CreateTable, DeleteTable, ListTables.
|
|
|
|
import pytest
|
|
from botocore.exceptions import ClientError
|
|
from util import list_tables, test_table_name, create_test_table, random_string
|
|
|
|
# Utility function for create a table with a given name and some valid
|
|
# schema.. This function initiates the table's creation, but doesn't
|
|
# wait for the table to actually become ready.
|
|
def create_table(dynamodb, name, BillingMode='PAY_PER_REQUEST', **kwargs):
|
|
return dynamodb.create_table(
|
|
TableName=name,
|
|
BillingMode=BillingMode,
|
|
KeySchema=[
|
|
{
|
|
'AttributeName': 'p',
|
|
'KeyType': 'HASH'
|
|
},
|
|
{
|
|
'AttributeName': 'c',
|
|
'KeyType': 'RANGE'
|
|
}
|
|
],
|
|
AttributeDefinitions=[
|
|
{
|
|
'AttributeName': 'p',
|
|
'AttributeType': 'S'
|
|
},
|
|
{
|
|
'AttributeName': 'c',
|
|
'AttributeType': 'S'
|
|
},
|
|
],
|
|
**kwargs
|
|
)
|
|
|
|
# Utility function for creating a table with a given name, and then deleting
|
|
# it immediately, waiting for these operations to complete. Since the wait
|
|
# uses DescribeTable, this function requires all of CreateTable, DescribeTable
|
|
# and DeleteTable to work correctly.
|
|
# Note that in DynamoDB, table deletion takes a very long time, so tests
|
|
# successfully using this function are very slow.
|
|
def create_and_delete_table(dynamodb, name, **kwargs):
|
|
table = create_table(dynamodb, name, **kwargs)
|
|
table.meta.client.get_waiter('table_exists').wait(TableName=name)
|
|
table.delete()
|
|
table.meta.client.get_waiter('table_not_exists').wait(TableName=name)
|
|
|
|
##############################################################################
|
|
|
|
# Test creating a table, and then deleting it, waiting for each operation
|
|
# to have completed before proceeding. Since the wait uses DescribeTable,
|
|
# this tests requires all of CreateTable, DescribeTable and DeleteTable to
|
|
# function properly in their basic use cases.
|
|
# Unfortunately, this test is extremely slow with DynamoDB because deleting
|
|
# a table is extremely slow until it really happens.
|
|
def test_create_and_delete_table(dynamodb):
|
|
create_and_delete_table(dynamodb, 'alternator_test')
|
|
|
|
# Test that recreating a table right after deleting it works without issues
|
|
def test_recreate_table(dynamodb):
|
|
create_and_delete_table(dynamodb, 'alternator_recr_test')
|
|
create_and_delete_table(dynamodb, 'alternator_recr_test')
|
|
|
|
# DynamoDB documentation specifies that table names must be 3-255 characters,
|
|
# and match the regex [a-zA-Z0-9._-]+. Names not matching these rules should
|
|
# be rejected, and no table be created.
|
|
def test_create_table_unsupported_names(dynamodb):
|
|
from botocore.exceptions import ParamValidationError, ClientError
|
|
# Intererstingly, the boto library tests for names shorter than the
|
|
# minimum length (3 characters) immediately, and failure results in
|
|
# ParamValidationError. But the other invalid names are passed to
|
|
# DynamoDB, which returns an HTTP response code, which results in a
|
|
# CientError exception.
|
|
with pytest.raises(ParamValidationError):
|
|
create_table(dynamodb, 'n')
|
|
with pytest.raises(ParamValidationError):
|
|
create_table(dynamodb, 'nn')
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
create_table(dynamodb, 'n' * 256)
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
create_table(dynamodb, 'nyh@test')
|
|
|
|
# On the other hand, names following the above rules should be accepted. Even
|
|
# names which the Scylla rules forbid, such as a name starting with .
|
|
def test_create_and_delete_table_non_scylla_name(dynamodb):
|
|
create_and_delete_table(dynamodb, '.alternator_test')
|
|
|
|
# names with 255 characters are allowed in Dynamo, but they are not currently
|
|
# supported in Scylla because we create a directory whose name is the table's
|
|
# name followed by 33 bytes (underscore and UUID). So currently, we only
|
|
# correctly support names with length up to 222.
|
|
def test_create_and_delete_table_very_long_name(dynamodb):
|
|
# In the future, this should work:
|
|
#create_and_delete_table(dynamodb, 'n' * 255)
|
|
# But for now, only 222 works:
|
|
create_and_delete_table(dynamodb, 'n' * 222)
|
|
# We cannot test the following on DynamoDB because it will succeed
|
|
# (DynamoDB allows up to 255 bytes)
|
|
#with pytest.raises(ClientError, match='ValidationException'):
|
|
# create_table(dynamodb, 'n' * 223)
|
|
|
|
# Tests creating a table with an invalid schema should return a
|
|
# ValidationException error.
|
|
def test_create_table_invalid_schema(dynamodb):
|
|
# The name of the table "created" by this test shouldn't matter, the
|
|
# creation should not succeed anyway.
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
dynamodb.create_table(
|
|
TableName='name_doesnt_matter',
|
|
BillingMode='PAY_PER_REQUEST',
|
|
KeySchema=[
|
|
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
|
|
{ 'AttributeName': 'c', 'KeyType': 'HASH' }
|
|
],
|
|
AttributeDefinitions=[
|
|
{ 'AttributeName': 'p', 'AttributeType': 'S' },
|
|
{ 'AttributeName': 'c', 'AttributeType': 'S' },
|
|
],
|
|
)
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
dynamodb.create_table(
|
|
TableName='name_doesnt_matter',
|
|
BillingMode='PAY_PER_REQUEST',
|
|
KeySchema=[
|
|
{ 'AttributeName': 'p', 'KeyType': 'RANGE' },
|
|
{ 'AttributeName': 'c', 'KeyType': 'RANGE' }
|
|
],
|
|
AttributeDefinitions=[
|
|
{ 'AttributeName': 'p', 'AttributeType': 'S' },
|
|
{ 'AttributeName': 'c', 'AttributeType': 'S' },
|
|
],
|
|
)
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
dynamodb.create_table(
|
|
TableName='name_doesnt_matter',
|
|
BillingMode='PAY_PER_REQUEST',
|
|
KeySchema=[
|
|
{ 'AttributeName': 'c', 'KeyType': 'RANGE' }
|
|
],
|
|
AttributeDefinitions=[
|
|
{ 'AttributeName': 'c', 'AttributeType': 'S' },
|
|
],
|
|
)
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
dynamodb.create_table(
|
|
TableName='name_doesnt_matter',
|
|
BillingMode='PAY_PER_REQUEST',
|
|
KeySchema=[
|
|
{ 'AttributeName': 'c', 'KeyType': 'HASH' },
|
|
{ 'AttributeName': 'p', 'KeyType': 'RANGE' },
|
|
{ 'AttributeName': 'z', 'KeyType': 'RANGE' }
|
|
],
|
|
AttributeDefinitions=[
|
|
{ 'AttributeName': 'c', 'AttributeType': 'S' },
|
|
{ 'AttributeName': 'p', 'AttributeType': 'S' },
|
|
{ 'AttributeName': 'z', 'AttributeType': 'S' }
|
|
],
|
|
)
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
dynamodb.create_table(
|
|
TableName='name_doesnt_matter',
|
|
BillingMode='PAY_PER_REQUEST',
|
|
KeySchema=[
|
|
{ 'AttributeName': 'c', 'KeyType': 'HASH' },
|
|
],
|
|
AttributeDefinitions=[
|
|
{ 'AttributeName': 'z', 'AttributeType': 'S' }
|
|
],
|
|
)
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
dynamodb.create_table(
|
|
TableName='name_doesnt_matter',
|
|
BillingMode='PAY_PER_REQUEST',
|
|
KeySchema=[
|
|
{ 'AttributeName': 'k', 'KeyType': 'HASH' },
|
|
],
|
|
AttributeDefinitions=[
|
|
{ 'AttributeName': 'k', 'AttributeType': 'Q' }
|
|
],
|
|
)
|
|
|
|
# Test that trying to create a table that already exists fails in the
|
|
# appropriate way (ResourceInUseException)
|
|
def test_create_table_already_exists(dynamodb, test_table):
|
|
with pytest.raises(ClientError, match='ResourceInUseException.*Table.*already exists'):
|
|
create_table(dynamodb, test_table.name)
|
|
|
|
# Test that BillingMode error path works as expected - only the values
|
|
# PROVISIONED or PAY_PER_REQUEST are allowed. The former requires
|
|
# ProvisionedThroughput to be set, the latter forbids it.
|
|
# If BillingMode is outright missing, it defaults (as original
|
|
# DynamoDB did) to PROVISIONED so ProvisionedThroughput is allowed.
|
|
def test_create_table_billing_mode_errors(dynamodb, test_table):
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
create_table(dynamodb, test_table_name(), BillingMode='unknown')
|
|
# billing mode is case-sensitive
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
create_table(dynamodb, test_table_name(), BillingMode='pay_per_request')
|
|
# PAY_PER_REQUEST cannot come with a ProvisionedThroughput:
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
create_table(dynamodb, test_table_name(),
|
|
BillingMode='PAY_PER_REQUEST', ProvisionedThroughput={'ReadCapacityUnits': 10, 'WriteCapacityUnits': 10})
|
|
# On the other hand, PROVISIONED requires ProvisionedThroughput:
|
|
# By the way, ProvisionedThroughput not only needs to appear, it must
|
|
# have both ReadCapacityUnits and WriteCapacityUnits - but we can't test
|
|
# this with boto3, because boto3 has its own verification that if
|
|
# ProvisionedThroughput is given, it must have the correct form.
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
create_table(dynamodb, test_table_name(), BillingMode='PROVISIONED')
|
|
# If BillingMode is completely missing, it defaults to PROVISIONED, so
|
|
# ProvisionedThroughput is required
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
dynamodb.create_table(TableName=test_table_name(),
|
|
KeySchema=[{ 'AttributeName': 'p', 'KeyType': 'HASH' }],
|
|
AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }])
|
|
|
|
# Even before Alternator gains full support for the DynamoDB stream API
|
|
# and CreateTable's StreamSpecification option, we should support the
|
|
# options which mean it is turned *off*.
|
|
def test_table_streams_off(dynamodb):
|
|
# If StreamSpecification is given, but has StreamEnabled=false, it's as
|
|
# if StreamSpecification was missing. StreamViewType isn't needed.
|
|
table = create_test_table(dynamodb, StreamSpecification={'StreamEnabled': False},
|
|
KeySchema=[{ 'AttributeName': 'p', 'KeyType': 'HASH' }],
|
|
AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }]);
|
|
table.delete();
|
|
# DynamoDB doesn't allow StreamSpecification to be empty map - if it
|
|
# exists, it must have a StreamEnabled
|
|
# Unfortunately, new versions of boto3 doesn't let us pass this...
|
|
#with pytest.raises(ClientError, match='ValidationException'):
|
|
# table = create_test_table(dynamodb, StreamSpecification={},
|
|
# KeySchema=[{ 'AttributeName': 'p', 'KeyType': 'HASH' }],
|
|
# AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }]);
|
|
# table.delete();
|
|
# Unfortunately, boto3 doesn't allow us to pass StreamSpecification=None.
|
|
# This is what we had in issue #5796.
|
|
|
|
# For tests with 'StreamEnabled': True, and different 'StreamViewType', see
|
|
# test_streams.py.
|
|
|
|
# Our first implementation had a special column name called "attrs" where
|
|
# we stored a map for all non-key columns. If the user tried to name one
|
|
# of the key columns with this same name, the result was a disaster - Scylla
|
|
# goes into a bad state after trying to write data with two updates to same-
|
|
# named columns.
|
|
special_column_name1 = 'attrs'
|
|
special_column_name2 = ':attrs'
|
|
@pytest.fixture(scope="session")
|
|
def test_table_special_column_name(dynamodb):
|
|
table = create_test_table(dynamodb,
|
|
KeySchema=[
|
|
{ 'AttributeName': special_column_name1, 'KeyType': 'HASH' },
|
|
{ 'AttributeName': special_column_name2, 'KeyType': 'RANGE' }
|
|
],
|
|
AttributeDefinitions=[
|
|
{ 'AttributeName': special_column_name1, 'AttributeType': 'S' },
|
|
{ 'AttributeName': special_column_name2, 'AttributeType': 'S' },
|
|
],
|
|
)
|
|
yield table
|
|
table.delete()
|
|
@pytest.mark.xfail(reason="special attrs column not yet hidden correctly")
|
|
def test_create_table_special_column_name(test_table_special_column_name):
|
|
s = random_string()
|
|
c = random_string()
|
|
h = random_string()
|
|
expected = {special_column_name1: s, special_column_name2: c, 'hello': h}
|
|
test_table_special_column_name.put_item(Item=expected)
|
|
got = test_table_special_column_name.get_item(Key={special_column_name1: s, special_column_name2: c}, ConsistentRead=True)['Item']
|
|
assert got == expected
|
|
|
|
# Test that all tables we create are listed, and pagination works properly.
|
|
# Note that the DyanamoDB setup we run this against may have hundreds of
|
|
# other tables, for all we know. We just need to check that the tables we
|
|
# created are indeed listed.
|
|
def test_list_tables_paginated(dynamodb, test_table, test_table_s, test_table_b):
|
|
my_tables_set = {table.name for table in [test_table, test_table_s, test_table_b]}
|
|
for limit in [1, 2, 3, 4, 50, 100]:
|
|
print("testing limit={}".format(limit))
|
|
list_tables_set = set(list_tables(dynamodb, limit))
|
|
assert my_tables_set.issubset(list_tables_set)
|
|
|
|
# Test that pagination limit is validated
|
|
def test_list_tables_wrong_limit(dynamodb):
|
|
# lower limit (min. 1) is imposed by boto3 library checks
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
dynamodb.meta.client.list_tables(Limit=101)
|
|
|
|
# Even before Alternator gains support for configuring server-side encryption
|
|
# ("encryption at rest") with CreateTable's SSESpecification option, we should
|
|
# support the option "Enabled=false" which is the default, and means the server
|
|
# takes care of whatever server-side encryption is done, on its own.
|
|
# Reproduces issue #7031.
|
|
def test_table_sse_off(dynamodb):
|
|
# If StreamSpecification is given, but has StreamEnabled=false, it's as
|
|
# if StreamSpecification was missing, and fine. No other attribues are
|
|
# necessary.
|
|
table = create_test_table(dynamodb, SSESpecification = {'Enabled': False},
|
|
KeySchema=[{ 'AttributeName': 'p', 'KeyType': 'HASH' }],
|
|
AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }]);
|
|
table.delete();
|