mirrors/scylladb
1
0
Fork 0
mirror of https://github.com/scylladb/scylladb.git synced 2026-04-20 16:40:35 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
28e59bae5acb4dc8b346fc2ca6ea99a6482659cd
scylladb/test/alternator/test_transact.py
Avi Kivity 0ae22a09d4 LICENSE: Update to version 1.1 
Updated terms of non-commercial use (must be a never-customer).
2026-04-12 19:46:33 +03:00

1080 lines
51 KiB
Python
Raw Blame History

# Copyright 2025-present ScyllaDB
#
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.1
# Tests for the multi-item transaction feature (issue #5064) -
# the TransactWriteItems and TransactGetItems requests.
#
# Note that the tests in this file check only the correct functionality of
# individual calls of these requests. They do not test the consistency or
# isolation guarantees, nor do they test the concurrent invocation of multiple
# requests. Such tests would require a different test framework, such as the
# one requested in issue #6350.
import pytest
from botocore.exceptions import ClientError
from boto3.dynamodb.types import TypeDeserializer
from .util import random_string
##########################################################################
# Test the basic functionality of the TransactWriteItems operation -
# Put, Delete, ConditionCheck, Update - on a single item of a single table.
# A ClientRequestToken is not passed in these basic tests.
##########################################################################
# Single Put action without a condition
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_put_unconditional(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': item
}}])
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Put action with a true condition - succeeds
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_put_true(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': item,
# The condition attribute_not_exists(p) is true for an
# item that doesn't exist
'ConditionExpression': 'attribute_not_exists(p)'
}}])
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Put action with a false condition fails with a
# TransactionCanceledException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_put_false(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': item,
# The condition attribute_exists(p) is false for an
# item that doesn't exist
'ConditionExpression': 'attribute_exists(p)'
}}])
# The transaction failed, so the item didn't go in:
assert 'Item' not in test_table_s.get_item(Key={'p': p}, ConsistentRead=True)
# Verify that a transaction's Put action, behaves like a PutItem request,
# and not like a CQL Insert, in that it completely replaces an existing item -
# it doesn't merge the new data into the existing item.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_put_replaces(test_table_s):
p = random_string()
x = random_string()
y = random_string()
test_table_s.put_item(Item={'p': p, 'x': x})
assert {'p': p, 'x': x} == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': {'p': p, 'y': y}
}}])
assert {'p': p, 'y': y} == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Delete action without a condition
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_delete_unconditional(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.put_item(Item=item)
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Delete': {
'TableName': test_table_s.name,
'Key': {'p': p}
}}])
# Item should be gone
assert 'Item' not in test_table_s.get_item(Key={'p': p}, ConsistentRead=True)
# Single Delete action with a true condition succeeds
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_delete_true(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.put_item(Item=item)
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Delete': {
'TableName': test_table_s.name,
'Key': {'p': p},
# The condition attribute_exists(p) is true for an
# item that exists
'ConditionExpression': 'attribute_exists(p)'
}}])
# Item should be gone
assert 'Item' not in test_table_s.get_item(Key={'p': p}, ConsistentRead=True)
# Single Delete action with a false condition fails with a
# TransactionCanceledException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_delete_false(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.put_item(Item=item)
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Delete': {
'TableName': test_table_s.name,
'Key': {'p': p},
# The condition attribute_not_exists(p) is false for an
# item that does exist
'ConditionExpression': 'attribute_not_exists(p)'
}}])
# The transaction failed, so the item wasn't deleted
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single ConditionCheck action without a condition is, unsurprisingly,
# not allowed - resulting in ValidationException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_conditioncheck_unconditional(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*[cC]onditionExpression'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p}
}}])
# Single ConditionCheck action with a true condition succeeds, but
# doesn't do anything (since it's just a condition check, not a write)
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_conditioncheck_true(test_table_s):
p = random_string()
test_table_s.meta.client.transact_write_items(TransactItems=[{
'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p},
# The condition attribute_not_exists(p) is true for an
# item that doesn't exist
'ConditionExpression': 'attribute_not_exists(p)'
}}])
# Single ConditionCheck action with a false condition fails with a
# TransactionCanceledException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_conditioncheck_false(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p},
# The condition attribute_exists(p) is false for an item that
# doesn't exist
'ConditionExpression': 'attribute_exists(p)'
}}])
# Unlike the previous test_transact_write_items_single_* tests which used
# trivial conditions without ExpressionAttributeNames and
# ExpressionAttributeValues, the following tests for Update actions do use
# those in UpdateExpression and/or ConditionExpression.
# Single Update action without a condition
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_update_unconditional(test_table_s):
p = random_string()
item = {'p': p, 'x': 42}
test_table_s.put_item(Item=item)
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'TableName': test_table_s.name,
'Key': {'p': p},
'UpdateExpression': 'SET #var = #var + :one',
'ExpressionAttributeNames': {'#var': 'x'},
'ExpressionAttributeValues': {':one': 1},
}}])
item['x'] += 1
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Update action with a true condition succeeds
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_update_true(test_table_s):
p = random_string()
item = {'p': p, 'x': 42}
test_table_s.put_item(Item=item)
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'TableName': test_table_s.name,
'Key': {'p': p},
# x is really 42, so this condition is true
'ConditionExpression': '#var = :fourtytwo',
'UpdateExpression': 'SET #var = #var + :one',
'ExpressionAttributeNames': {'#var': 'x'},
'ExpressionAttributeValues': {':one': 1, ':fourtytwo': 42},
}}])
item['x'] += 1
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Update action with a false condition fails with a
# TransactionCanceledException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_update_false(test_table_s):
p = random_string()
item = {'p': p, 'x': 42}
test_table_s.put_item(Item=item)
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'TableName': test_table_s.name,
'Key': {'p': p},
# x is really 42, not 43, so this condition is false
'ConditionExpression': '#var = :fourtythree',
'UpdateExpression': 'SET #var = #var + :one',
'ExpressionAttributeNames': {'#var': 'x'},
'ExpressionAttributeValues': {':one': 1, ':fourtythree': 43},
}}])
# Since the condition failed, the item remains unchanged
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
##########################################################################
# Additional tests for less basic TransactWriteItems functionality
##########################################################################
# Check that without ClientRequestToken, a request is not idempotent -
# if we increment the same counter twice it happens twice. But if we do
# use ClientRequestToken and pass the same one twice, only the first increment
# happens.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_clientrequesttoken(test_table_s):
p = random_string()
item = {'p': p, 'x': 42}
test_table_s.put_item(Item=item)
increment_x = [{
'Update': {
'TableName': test_table_s.name, 'Key': {'p': p},
'UpdateExpression': 'SET x = x + :one',
'ExpressionAttributeValues': {':one': 1},
}
}]
# Doing two transactions each incrementing x without ClientRequestToken,
# both happen:
test_table_s.meta.client.transact_write_items(
TransactItems=increment_x)
assert 43 == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['x']
test_table_s.meta.client.transact_write_items(
TransactItems=increment_x)
assert 44 == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['x']
# But if we pass a ClientRequestToken and attempt the same transaction
# twice, only one is performed. The second transaction succeeds - it
# just has no affect.
token = random_string()
test_table_s.meta.client.transact_write_items(
ClientRequestToken=token, TransactItems=increment_x)
assert 45 == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['x']
test_table_s.meta.client.transact_write_items(
ClientRequestToken=token, TransactItems=increment_x)
assert 45 == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['x']
# Check that ClientRequestToken can be a string between 1 and 36 characters,
# anything else is rejected.
# Note that there is no reliable way to check the 1 character case: If we
# repeat the same test twice in a 10 minute period and it uses a different
# table name, we can't reuse the same ClientRequestToken or we'll get a
# IdempotentParameterMismatch error. We must use a random token - and
# a random one-character string is not enough to avoid test flakiness.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_clientrequesttoken_length(test_table_s):
p = random_string()
item = {'p': p}
transaction = [{'Put': {'TableName': test_table_s.name, 'Item': item}}]
# 36 characters work
test_table_s.meta.client.transact_write_items(
ClientRequestToken=random_string(length=36),
TransactItems=transaction)
# 37 characters is too long for ClientRequestToken, a ValidationException
# results:
with pytest.raises(ClientError, match='ValidationException.*[cC]lientRequestToken'):
test_table_s.meta.client.transact_write_items(
ClientRequestToken=random_string(length=37),
TransactItems=transaction)
# Check that if the same ClientRequestToken is used for two transactions,
# but they are different, an IdempotentParameterMismatch error is thrown.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_clientrequesttoken_mismatch(test_table_s):
transaction1 = [{'Put': {'TableName': test_table_s.name, 'Item': {'p': random_string()}}}]
transaction2 = [{'Put': {'TableName': test_table_s.name, 'Item': {'p': random_string()}}}]
token = random_string()
test_table_s.meta.client.transact_write_items(
ClientRequestToken=token, TransactItems=transaction1)
with pytest.raises(ClientError, match='IdempotentParameterMismatchException'):
test_table_s.meta.client.transact_write_items(
ClientRequestToken=token, TransactItems=transaction2)
# All tests above involved a transaction with a single action. Here
# we finally begin to check trasactions with multiple actions. Let's
# begin with a successful TransactWriteItems transaction, where some of
# the actions have successful conditions, and some don't have conditions
# at all, and all the actions are performed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_multi_action_true(test_table_s):
p1 = random_string()
p2 = random_string()
p3 = random_string()
item1 = {'p': p1, 'x': 'dog'}
item2 = {'p': p2, 'x': 'cat'}
test_table_s.meta.client.transact_write_items(TransactItems=[
# unconditional Put
{ 'Put': {
'TableName': test_table_s.name,
'Item': item1
}},
# Put with true condition (attribute_not_exists(p) is true
# when the item doesn't exist).
{ 'Put': {
'TableName': test_table_s.name,
'Item': item2,
'ConditionExpression': 'attribute_not_exists(p)'
}},
# ConditionCheck with true condition. It is true, but doesn't
# cause anything to be written for key p3.
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p3},
'ConditionExpression': 'attribute_not_exists(p)'
}},
])
# p1 and p2 supposed to have been written by the transaction, but p3
# wasn't because it was only involved in a ConditionCheck.
assert item1 == test_table_s.get_item(Key={'p': p1}, ConsistentRead=True)['Item']
assert item2 == test_table_s.get_item(Key={'p': p2}, ConsistentRead=True)['Item']
assert 'Item' not in test_table_s.get_item(Key={'p': p3}, ConsistentRead=True)
# Test a transaction with several actions, one of which has a false
# condition. The entire transaction should fail, and none of its actions
# (not even those with true conditions or no conditions) should be performed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_multi_action_false(test_table_s):
p1 = random_string()
p2 = random_string()
p3 = random_string()
item1 = {'p': p1, 'x': 'dog'}
item2 = {'p': p2, 'x': 'cat'}
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[
# unconditional Put
{ 'Put': {
'TableName': test_table_s.name,
'Item': item1
}},
# Put with true condition (attribute_not_exists(p) is true
# when the item doesn't exist).
{ 'Put': {
'TableName': test_table_s.name,
'Item': item2,
'ConditionExpression': 'attribute_not_exists(p)'
}},
# ConditionCheck with a *false* condition. It should cause the
# entire transaction to be rejected.
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p3},
'ConditionExpression': 'attribute_exists(p)'
}},
])
# Because the entire transaction was rejected, none of its actions
# should have been done. Items for p1, p2 and p3 should all not exist.
assert 'Item' not in test_table_s.get_item(Key={'p': p1}, ConsistentRead=True)
assert 'Item' not in test_table_s.get_item(Key={'p': p2}, ConsistentRead=True)
assert 'Item' not in test_table_s.get_item(Key={'p': p3}, ConsistentRead=True)
# Test that it's not allowed for two actions in the same transaction to
# target the same item (this limitation also includes ConditionCheck
# actions).
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_multi_action_conflict(test_table_s):
p1 = random_string()
p2 = random_string()
with pytest.raises(ClientError, match='ValidationException.*one item'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}},
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}}])
with pytest.raises(ClientError, match='ValidationException.*one item'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}},
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p1},
'ConditionExpression': 'attribute_not_exists(p)'
}}])
with pytest.raises(ClientError, match='ValidationException.*one item'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}},
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p2},
'ConditionExpression': 'attribute_not_exists(p)'
}},
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p2},
'ConditionExpression': 'attribute_not_exists(p)'
}}])
# Test that a transaction may involve more than one table, not just more than
# one item.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_multi_table_true(test_table_s, test_table_ss):
p1 = random_string()
p2 = random_string()
c2 = random_string()
item1 = {'p': p1, 'x': 'dog'}
item2 = {'p': p2, 'c': c2, 'x': 'cat'}
test_table_s.meta.client.transact_write_items(TransactItems=[
# unconditional Put on the first table
{ 'Put': {
'TableName': test_table_s.name,
'Item': item1
}},
# Put with true condition on the second table
{ 'Put': {
'TableName': test_table_ss.name,
'Item': item2,
'ConditionExpression': 'attribute_not_exists(p)'
}}
])
# p1 and p2 supposed to have been written by the transaction on the
# two different tables
assert item1 == test_table_s.get_item(Key={'p': p1}, ConsistentRead=True)['Item']
assert item2 == test_table_ss.get_item(Key={'p': p2, 'c': c2}, ConsistentRead=True)['Item']
# Check that a TransactWriteItems with zero items is not allowed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_empty(test_table_s):
with pytest.raises(ClientError, match='ValidationException.*[tT]ransactItems'):
test_table_s.meta.client.transact_write_items(TransactItems=[])
# Check that a TransactWriteItems with 100 items is allowed. The limit
# used to be just 25 items, but it was increased to 100 in September 2022.
# The next test will check that *more* than 100 items is not allowed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_100(test_table_s):
p = random_string()
items = [{'p': p + str(i), 'x': i} for i in range(100)]
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': item,
}} for item in items])
# verify that all 100 items went in
for item in items:
assert item == test_table_s.get_item(Key={'p': item['p']}, ConsistentRead=True)['Item']
# Check that a transaction with 101 (>100) items is rejected.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_101(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*[tT]ransactItems.*100'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p + str(i)},
}} for i in range(101)])
# Beyond limiting a TransactWriteItems to 100 items (verified in tests above)
# DynamoDB has an aditional limit that "the aggregate size of the items in the
# transaction cannot exceed 4 MB". We can't write 5 MB in a transaction even
# if we build it from relatively small items (each not exceeding the 400 KB
# limit the size of an individual item size).
# Note that this test checks only Put actions which include the entire
# new item in the transaction - so the transaction itself reaches 5MB in
# size. In the next test we will check what happens for Update or Delete
# actions which may themselves have small size but refer to large items.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_put_5MB(test_table_s):
p = random_string()
# We will write 50 items of roughly 100KB each, reaching around 5MB
long = 'x'*100000
items = [{'p': p + str(i), 'x': long} for i in range(50)]
with pytest.raises(ClientError, match='ValidationException.*size cannot exceed'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': item,
}} for item in items])
# In the previous test (test_transact_write_items_put_5MB) we saw that we
# can't have over 4 MB of new data in a transaction. But the DynamoDB
# TransactWriteItems documentation is ambiguous on what this 4 MB limit
# applies to: The documentation says that it's not allowed that "the aggregate
# size of the items in the transaction exceeds 4 MB". Is this the size of the
# items *mentioned* in the transaction, or the size of items *in* the
# transaction? In other words, if we have a transaction doing Deletes of
# large items - where the transaction itself is very small but the total item
# size exceeds 4 MB - is this allowed or not?
# It turns out that the answer is that the size of the transaction request
# (DynamoDB refers to it as "transaction payload" in the error messages) is
# what matters, not the size of the items. A delete transaction that deletes
# 5 MB of items but the transaction itself is small - is allowed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_delete_5MB(test_table_s):
p = random_string()
# Write (not in a transaction) 50 items of roughly 100 KB each, so their
# total size is roughly 5 MB.
long = 'x'*100000
items = [{'p': p + str(i), 'x': long} for i in range(50)]
with test_table_s.batch_writer() as batch:
for item in items:
batch.put_item(item)
# Now try to delete those 50 items in one transaction. The transaction
# request is very small, but the total size of the items to be deleted
# is around 5 MB. This transaction *is* allowed showing that DynamoDB's
# limit is really on the transaction size, not the "aggregate size of the
# items" mentioned in the documentation.
try:
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Delete': {
'TableName': test_table_s.name,
'Key': {'p': item['p']},
}} for item in items])
# Verify the deletes worked
for item in items:
assert 'Item' not in test_table_s.get_item(Key={'p': item['p']}, ConsistentRead=True)
except ClientError as e:
# A 5 MB write transaction takes roughly 10,000 WCUs, which DynamoDB
# sometimes refuses to do on a brand-new on-demand table, so the
# transaction gets rejected. But if this happens, it's an expected
# failure. It's not a real failure (a real failure would have been
# the transaction failing on something other than ThrottlingException).
if e.response['Error']['Code'] == 'ThrottlingException':
pytest.xfail()
else:
raise
# But verify that aggregate transaction size of 3MB is fine. Note that
# individual items are limited to 400KB, so we must a transaction with
# several smaller items.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_put_3MB(test_table_s):
p = random_string()
# We will write 30 items of roughly 100KB each, reaching around 3MB
long = 'x'*100000
items = [{'p': p + str(i), 'x': long} for i in range(30)]
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': item,
}} for item in items])
# verify that all 30 items went in
for item in items:
assert item == test_table_s.get_item(Key={'p': item['p']}, ConsistentRead=True)['Item']
# In various tests above we checked that failed conditions result in a
# TransactionCanceledException. But this error comes with CancellationReasons
# and in the following test we want to test these.
# Note that this test only checks the cancellation reasons "None" and
# "ConditionalCheckFailed". The test after it will check the "ValidationError"
# reason. There are additional reasons, but these tests are not designed to
# be able to reach those cases:
# 1. ItemCollectionSizeLimitExceeded - when there is an LSI and the write
# caused the item collection (Scylla partition) to go over 10 GB.
# 2. TransactionConflict - when there is a conflict with another transaction.
# As explained above, we don't test concurrent transactions in this test
# suite.
# 3. ProvisionedThroughputExceeded - self-explanatory, not tested in these
# tests.
# 4. ThrottlingError - similar, for on-demand tables that haven't scaled
# enough yet.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_cancellation_reasons_conditionalcheckfailed(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
# The condition attribute_exists(p) is false for an
# item that doesn't exist
'ConditionExpression': 'attribute_exists(p)'
}}])
# The error object should have a CancellationReasons member. Our
# transaction has one item, and one failure, so the CancellationReasons
# should be a one-member array. This member has a code, and a message.
assert 'CancellationReasons' in e.value.response
reasons = e.value.response['CancellationReasons']
assert len(reasons) == 1
assert reasons[0]['Code'] == 'ConditionalCheckFailed'
# The Message is a user-readable message like "The conditional request
# failed". We'll assert it exists, but not insist what the text is
assert 'Message' in reasons[0]
# Now do the same thing with a transaction with multiple actions, some
# with failed conditions so the entire transaction gets canceled.
# We'll see that the CancellationReasons lists each one of the actions
# in order, and says which ones had a failed condition.
p1 = random_string()
p2 = random_string()
p3 = random_string()
p4 = random_string()
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[
# Unconditional Put
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}},
# Put with true condition (attribute_not_exists(p) is true
# when the item doesn't exist).
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p2},
'ConditionExpression': 'attribute_not_exists(p)'
}},
# ConditionCheck with a *false* condition. It should cause the
# entire transaction to be rejected.
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p3},
'ConditionExpression': 'attribute_exists(p)'
}},
# Another false condition, on a Put action:
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p4},
'ConditionExpression': 'attribute_exists(p)'
}},
])
assert 'CancellationReasons' in e.value.response
reasons = e.value.response['CancellationReasons']
# The CancellationReasons are listed in the same order of the actions
# in the transaction - we expect the first two actions to have not
# failed, so they have Code="None", and the third and forth both failed.
# It's interesting that DynamoDB does report both failures - and doesn't
# "short circuit" after the first one.
assert len(reasons) == 4
assert reasons[0]['Code'] == 'None'
assert reasons[1]['Code'] == 'None'
assert reasons[2]['Code'] == 'ConditionalCheckFailed'
assert reasons[3]['Code'] == 'ConditionalCheckFailed'
# Actions "None" reason don't come with a user-readable message,
# but others do.
assert 'Message' not in reasons[0]
assert 'Message' not in reasons[1]
assert 'Message' in reasons[2]
assert 'Message' in reasons[3]
# Another possible cancellation reason is a ValidationError in one of the
# actions - for example an attempt to write a wrong type into a key or
# exceeding the size limit on a key or the size of an item, or exceeding the
# nesting level limits.
# Note that the actions are validated before any of them actually run,
# so if any action has a ValidationError, other correctly-validated
# actions will return a "None" reason - even if potentially they should
# have been a false condition.
# In fact it turns out (and we'll confirm in this test) that if two actions
# have a validation error, only the first error is detected, and the server
# doesn't continue to find the validation error in the second action. It's
# probably not important that Alternator be compatible with this behavior,
# but if we can, why not.
#
# Surprisingly, many validations actually return a ValidationException
# on the entire transaction instead of a TransactionCanceledException with
# one ValidationError in its cancellation reason. For example (we have a
# test for this below), a reference to a missing ExpressionAttributeNames.
# It's not clear how DynamoDB decided which case will return a
# ValidationException and which a ValidationError, but let's be compatible
# with what DynamoDB does.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_cancellation_reasons_validationerror(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
# The key p should be a string, not a number, so this
# item has a validation error.
'Item': {'p': 7}
}},
{ 'Put': {
'TableName': test_table_s.name,
# A second validation error. As explained above, as soon
# as the first validation error was found above, this action
# will not be reached and its validation error will not be
# detected. So we expect "None" reason on this action.
'Item': {'p': 8}
}},
{ 'Put': {
'TableName': test_table_s.name,
# This condition is false (the item does *not* exist yet),
# and would have failed the tranaction, but the server won't
# even get around to checking that because of the validation
# error in the other actions. So we will expect to get a
# "None" reason on this action.
'ConditionExpression': 'attribute_exists(p)',
'Item': {'p': p}
}},
])
assert 'CancellationReasons' in e.value.response
reasons = e.value.response['CancellationReasons']
assert len(reasons) == 3
assert reasons[0]['Code'] == 'ValidationError'
assert reasons[1]['Code'] == 'None'
assert reasons[2]['Code'] == 'None'
assert 'Message' in reasons[0]
# "None" reason doesn't come with a message:
assert 'Message' not in reasons[1]
assert 'Message' not in reasons[2]
# Verify that even if there is just *one* action in the transaction,
# on certain types of errors like the incorrect key type we used above
# in test_transact_write_cancellation_reasons_validationerror will get a
# TransactionCanceledException with a single ValidationError - not a
# ValidationException. Below we'll see in other tests that this is not
# always true - in some other types of errors, we actually do get a
# ValidationException.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_cancellation_reasons_validationerror_one(test_table_s):
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
# The key p should be a string, not a number, so this
# item has a validation error.
'Item': {'p': 7}
}}])
reasons = e.value.response['CancellationReasons']
assert reasons[0]['Code'] == 'ValidationError'
assert 'Message' in reasons[0]
# Check the ReturnValuesOnConditionCheckFailure parameter, which allows
# the user to retrieve the content of the item that caused a specific
# condition to fail.
# Note that ReturnValuesOnConditionCheckFailure is specified for a
# specific action, not for the entire transaction.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_returnvaluesonconditioncheckfailure(test_table_s):
p = random_string()
item = {'p': p, 'x': 42, 'y': 'dog'}
test_table_s.put_item(Item=item)
# Check ReturnValuesOnConditionCheckFailure="ALL_OLD"
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'ReturnValuesOnConditionCheckFailure': 'ALL_OLD',
'TableName': test_table_s.name,
'Key': {'p': p},
# x is really 42, not 43, so this condition is false
'ConditionExpression': 'x = :fourtythree',
'UpdateExpression': 'SET x = x + :one',
'ExpressionAttributeValues': {':one': 1, ':fourtythree': 43},
}}])
# The relevant CancellationReasons will now have an "Item".
# The returned reasons[0]['Item'] has DynamoDB JSON encoding, and in
# this specific place (inside the error object) boto3 forgot to
# deserialize the JSON encoding for us (like it does automatically
# in other place) - so we need to do this deserialization manually
# with TypeDeserializer before we can compare it to the expected "item".
assert 'CancellationReasons' in e.value.response
reasons = e.value.response['CancellationReasons']
assert len(reasons) == 1
assert reasons[0]['Code'] == 'ConditionalCheckFailed'
assert 'Message' in reasons[0]
deserializer = TypeDeserializer()
got_item = {x:deserializer.deserialize(y) for (x,y) in reasons[0]['Item'].items()}
assert item == got_item
# The same transaction with ReturnValuesOnConditionCheckFailure="NONE"
# doesn't return the Item:
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'ReturnValuesOnConditionCheckFailure': 'NONE',
'TableName': test_table_s.name,
'Key': {'p': p},
'ConditionExpression': 'x = :fourtythree',
'UpdateExpression': 'SET x = x + :one',
'ExpressionAttributeValues': {':one': 1, ':fourtythree': 43},
}}])
reasons = e.value.response['CancellationReasons']
assert len(reasons) == 1
assert reasons[0]['Code'] == 'ConditionalCheckFailed'
assert 'Item' not in reasons[0]
# Setting ReturnValuesOnConditionCheckFailure to anything else, e.g.,
# lowercase 'none', or 'dog', is a validation error. Surprisingly,
# even though ReturnValuesOnConditionCheckFailure is per-action, and
# DynamoDB can return TransactionCanceledException with a per-action
# ValidationError, in this case it doesn't do that - it returns a
# ValidationException for the entire request.
for option in ['none', 'dog']:
with pytest.raises(ClientError, match='ValidationException.*[rR]eturnValuesOnConditionCheckFailure'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'ReturnValuesOnConditionCheckFailure': option,
'TableName': test_table_s.name,
'Key': {'p': p},
'ConditionExpression': 'x = :fourtythree',
'UpdateExpression': 'SET x = x + :one',
'ExpressionAttributeValues': {':one': 1, ':fourtythree': 43},
}}])
# Various checks for ExpressionAttributeValues and ExpressionAttributeNames
# in a ConditionExpression, such as missing or unused entries in those arrays.
# We already have tests for ConditionExpression in test_condition_expression.py
# but here the problem happens inside one action instead of the whole
# transaction so we need to check if it's also recognized, and whether it
# causes a TransactionCanceledException with one ValidationError or a
# ValidationException on the entire transaction. It turns out that it's
# the latter.
# Check ConditionExpression reference to missing ExpressionAttributeNames
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_missing_name(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*#xyz'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
# The reference "#xyz" is missing in ExpressionAttributeNames
'ConditionExpression': 'attribute_exists(#xyz)'
}}])
# Check ConditionExpression value missing in ExpressionAttributeValues
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_missing_value(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*:xyz'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
# The reference ":xyz" is missing in ExpressionAttributeValues
'ConditionExpression': 'p = :xyz'
}}])
# Check unused name in ExpressionAttributeName
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_unused_name(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*unused.*#xyz'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
'ConditionExpression': 'attribute_exists(p)',
# The name "#xyz" isn't used in in any expression
'ExpressionAttributeNames': {'#xyz': 'x'}
}}])
# Check unused value in ExpressionAttributeValues
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_unused_value(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*unused.*:xyz'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
'ConditionExpression': 'attribute_exists(p)',
# The value ":xyz" isn't used in in any expression
'ExpressionAttributeValues': {':xyz': 7}
}}])
# Syntax error in a ConditionExpression in one action also returns a
# ValidationException for the entire transaction, not per item:
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_syntax_error(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*Syntax error'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
# "hello!" is a syntax error as an expression
'ConditionExpression': 'hello!',
}}])
# Some other errors in a ConditionExpression, like an ExpressionAttributeNames
# with an integer instead of a string (an attribute name), can generate a
# a SerializationException instead of a ValidationException. It's not clear
# to me why this is important, and I think it's fine if Alternator would
# return a ValidationException here and we change the test to accept both.
# But the important point is that the single exception is returned for the
# entire transaction - not per item.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_serialization_exception(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='SerializationException'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
'ConditionExpression': 'attribute_exists(p)',
# ExpressionAttributeNames expects strings (names of
# attributes), not the integer 7.
'ExpressionAttributeNames': {'#xyz': 7}
}}])
##########################################################################
# Tests for TransactGetItems functionality
##########################################################################
# Test basic transaction with one "Get" action
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_one(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.put_item(Item=item)
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
}}])
assert 'Responses' in ret
assert len(ret['Responses']) == 1
assert 'Item' in ret['Responses'][0]
assert item == ret['Responses'][0]['Item']
# If a Get transaction can't find an item with the given key, it's not
# an error - one of the Responses entries will just not have an "Item":
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_missing(test_table_s):
p = random_string()
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
}}])
assert 'Responses' in ret
assert len(ret['Responses']) == 1
assert 'Item' not in ret['Responses'][0]
# Test the ProjectionExpression parameter for a Get action, asking not to
# get the entire item and rather just get specific attributes. See more
# extensive tests for ProjectionExpression in test_projection_expression.py.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_projection_expression(test_table_s):
p = random_string()
item = {'p': p, 'x': 1, 'y': 2, 'z': 3}
test_table_s.put_item(Item=item)
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
'ProjectionExpression': 'x,z'
}}])
assert 'Responses' in ret
assert len(ret['Responses']) == 1
assert 'Item' in ret['Responses'][0]
assert {'x': item['x'], 'z': item['z']} == ret['Responses'][0]['Item']
# ProjectionExpression also supports ExpressionAttributeNames.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_projection_expression_attribute_names(test_table_s):
p = random_string()
item = {'p': p, 'x': 1, 'y': 2, 'z': 3}
test_table_s.put_item(Item=item)
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
'ProjectionExpression': '#xx,#zz',
'ExpressionAttributeNames': {'#xx': 'x', '#zz': 'z'}
}}])
assert 'Responses' in ret
assert len(ret['Responses']) == 1
assert 'Item' in ret['Responses'][0]
assert {'x': item['x'], 'z': item['z']} == ret['Responses'][0]['Item']
# If ExpressionAttributeNames is missing a name, or has an unused name,
# it's an error. As we saw above in other cases, it's a ValidationException
# for the entire transaction - not a TransactionCanceledException.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_unused_expressionattributenames(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*unused.*#qq'):
test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
'ProjectionExpression': '#xx,#zz',
# The reference "#qq" isn't used in the ProjectionExpression
'ExpressionAttributeNames': {'#xx': 'x', '#zz': 'z', '#qq': 'q'}
}}])
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_missing_expressionattributenames(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*#zz'):
test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
'ProjectionExpression': '#xx,#zz',
# The reference "#zz" is missing in ExpressionAttributeNames
'ExpressionAttributeNames': {'#xx': 'x'}
}}])
# Test the ability to read multiple items in one transaction. We can actually
# read 100 small items in one transaction - the limit used to be just 25
# items, but it was increased to 100 in September 2022 so let's verify that
# it works.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_100(test_table_s):
p = random_string()
items = [{'p': p + str(i), 'x': i} for i in range(100)]
with test_table_s.batch_writer() as batch:
for item in items:
batch.put_item(item)
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': item['p']},
}} for item in items])
# verify that all 100 items were read
for item in items:
assert item == test_table_s.get_item(Key={'p': item['p']}, ConsistentRead=True)['Item']
assert 'Responses' in ret
assert len(ret['Responses']) == 100
for i, response in enumerate(ret['Responses']):
assert 'Item' in response
assert response['Item'] == items[i]
# A transaction with 100 read actions is the limit, and 101 are not allowed:
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_101(test_table_s):
with pytest.raises(ClientError, match='ValidationException.*[tT]ransactItems.*100'):
test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': str(i)},
}} for i in range(101)])
Reference in New Issue View Git Blame Copy Permalink