mirrors/scylladb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
master
scylladb/test/alternator/test_transact.py
Nadav Har'El da00401b7d test/alternator: rename test with duplicate name 
The file test/alternator/test_transact.py accidentally had two tests
with the same name, test_transact_get_items_projection_expression.
This means the first of the two tests was ignored and never run.

This patch renames the second of the two to a more appropriate
(and unique...) name.

I verified that after this change the number of tests in this file
grows by one, and that still all tests pass on DynamoDB and fail
(as expected by xfail) on Alternator.

Signed-off-by: Nadav Har'El <nyh@scylladb.com>

Closes scylladb/scylladb#27702
2025-12-24 13:43:43 +02:00

1080 lines
51 KiB
Python
Raw Permalink Blame History

# Copyright 2025-present ScyllaDB
#
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
# Tests for the multi-item transaction feature (issue #5064) -
# the TransactWriteItems and TransactGetItems requests.
#
# Note that the tests in this file check only the correct functionality of
# individual calls of these requests. They do not test the consistency or
# isolation guarantees, nor do they test the concurrent invocation of multiple
# requests. Such tests would require a different test framework, such as the
# one requested in issue #6350.
import pytest
from botocore.exceptions import ClientError
from boto3.dynamodb.types import TypeDeserializer
from .util import random_string
##########################################################################
# Test the basic functionality of the TransactWriteItems operation -
# Put, Delete, ConditionCheck, Update - on a single item of a single table.
# A ClientRequestToken is not passed in these basic tests.
##########################################################################
# Single Put action without a condition
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_put_unconditional(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': item
}}])
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Put action with a true condition - succeeds
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_put_true(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': item,
# The condition attribute_not_exists(p) is true for an
# item that doesn't exist
'ConditionExpression': 'attribute_not_exists(p)'
}}])
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Put action with a false condition fails with a
# TransactionCanceledException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_put_false(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': item,
# The condition attribute_exists(p) is false for an
# item that doesn't exist
'ConditionExpression': 'attribute_exists(p)'
}}])
# The transaction failed, so the item didn't go in:
assert 'Item' not in test_table_s.get_item(Key={'p': p}, ConsistentRead=True)
# Verify that a transaction's Put action, behaves like a PutItem request,
# and not like a CQL Insert, in that it completely replaces an existing item -
# it doesn't merge the new data into the existing item.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_put_replaces(test_table_s):
p = random_string()
x = random_string()
y = random_string()
test_table_s.put_item(Item={'p': p, 'x': x})
assert {'p': p, 'x': x} == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': {'p': p, 'y': y}
}}])
assert {'p': p, 'y': y} == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Delete action without a condition
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_delete_unconditional(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.put_item(Item=item)
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Delete': {
'TableName': test_table_s.name,
'Key': {'p': p}
}}])
# Item should be gone
assert 'Item' not in test_table_s.get_item(Key={'p': p}, ConsistentRead=True)
# Single Delete action with a true condition succeeds
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_delete_true(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.put_item(Item=item)
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Delete': {
'TableName': test_table_s.name,
'Key': {'p': p},
# The condition attribute_exists(p) is true for an
# item that exists
'ConditionExpression': 'attribute_exists(p)'
}}])
# Item should be gone
assert 'Item' not in test_table_s.get_item(Key={'p': p}, ConsistentRead=True)
# Single Delete action with a false condition fails with a
# TransactionCanceledException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_delete_false(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.put_item(Item=item)
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Delete': {
'TableName': test_table_s.name,
'Key': {'p': p},
# The condition attribute_not_exists(p) is false for an
# item that does exist
'ConditionExpression': 'attribute_not_exists(p)'
}}])
# The transaction failed, so the item wasn't deleted
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single ConditionCheck action without a condition is, unsurprisingly,
# not allowed - resulting in ValidationException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_conditioncheck_unconditional(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*[cC]onditionExpression'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p}
}}])
# Single ConditionCheck action with a true condition succeeds, but
# doesn't do anything (since it's just a condition check, not a write)
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_conditioncheck_true(test_table_s):
p = random_string()
test_table_s.meta.client.transact_write_items(TransactItems=[{
'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p},
# The condition attribute_not_exists(p) is true for an
# item that doesn't exist
'ConditionExpression': 'attribute_not_exists(p)'
}}])
# Single ConditionCheck action with a false condition fails with a
# TransactionCanceledException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_conditioncheck_false(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p},
# The condition attribute_exists(p) is false for an item that
# doesn't exist
'ConditionExpression': 'attribute_exists(p)'
}}])
# Unlike the previous test_transact_write_items_single_* tests which used
# trivial conditions without ExpressionAttributeNames and
# ExpressionAttributeValues, the following tests for Update actions do use
# those in UpdateExpression and/or ConditionExpression.
# Single Update action without a condition
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_update_unconditional(test_table_s):
p = random_string()
item = {'p': p, 'x': 42}
test_table_s.put_item(Item=item)
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'TableName': test_table_s.name,
'Key': {'p': p},
'UpdateExpression': 'SET #var = #var + :one',
'ExpressionAttributeNames': {'#var': 'x'},
'ExpressionAttributeValues': {':one': 1},
}}])
item['x'] += 1
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Update action with a true condition succeeds
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_update_true(test_table_s):
p = random_string()
item = {'p': p, 'x': 42}
test_table_s.put_item(Item=item)
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'TableName': test_table_s.name,
'Key': {'p': p},
# x is really 42, so this condition is true
'ConditionExpression': '#var = :fourtytwo',
'UpdateExpression': 'SET #var = #var + :one',
'ExpressionAttributeNames': {'#var': 'x'},
'ExpressionAttributeValues': {':one': 1, ':fourtytwo': 42},
}}])
item['x'] += 1
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# Single Update action with a false condition fails with a
# TransactionCanceledException
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_single_update_false(test_table_s):
p = random_string()
item = {'p': p, 'x': 42}
test_table_s.put_item(Item=item)
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'TableName': test_table_s.name,
'Key': {'p': p},
# x is really 42, not 43, so this condition is false
'ConditionExpression': '#var = :fourtythree',
'UpdateExpression': 'SET #var = #var + :one',
'ExpressionAttributeNames': {'#var': 'x'},
'ExpressionAttributeValues': {':one': 1, ':fourtythree': 43},
}}])
# Since the condition failed, the item remains unchanged
assert item == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']
##########################################################################
# Additional tests for less basic TransactWriteItems functionality
##########################################################################
# Check that without ClientRequestToken, a request is not idempotent -
# if we increment the same counter twice it happens twice. But if we do
# use ClientRequestToken and pass the same one twice, only the first increment
# happens.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_clientrequesttoken(test_table_s):
p = random_string()
item = {'p': p, 'x': 42}
test_table_s.put_item(Item=item)
increment_x = [{
'Update': {
'TableName': test_table_s.name, 'Key': {'p': p},
'UpdateExpression': 'SET x = x + :one',
'ExpressionAttributeValues': {':one': 1},
}
}]
# Doing two transactions each incrementing x without ClientRequestToken,
# both happen:
test_table_s.meta.client.transact_write_items(
TransactItems=increment_x)
assert 43 == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['x']
test_table_s.meta.client.transact_write_items(
TransactItems=increment_x)
assert 44 == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['x']
# But if we pass a ClientRequestToken and attempt the same transaction
# twice, only one is performed. The second transaction succeeds - it
# just has no affect.
token = random_string()
test_table_s.meta.client.transact_write_items(
ClientRequestToken=token, TransactItems=increment_x)
assert 45 == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['x']
test_table_s.meta.client.transact_write_items(
ClientRequestToken=token, TransactItems=increment_x)
assert 45 == test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['x']
# Check that ClientRequestToken can be a string between 1 and 36 characters,
# anything else is rejected.
# Note that there is no reliable way to check the 1 character case: If we
# repeat the same test twice in a 10 minute period and it uses a different
# table name, we can't reuse the same ClientRequestToken or we'll get a
# IdempotentParameterMismatch error. We must use a random token - and
# a random one-character string is not enough to avoid test flakiness.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_clientrequesttoken_length(test_table_s):
p = random_string()
item = {'p': p}
transaction = [{'Put': {'TableName': test_table_s.name, 'Item': item}}]
# 36 characters work
test_table_s.meta.client.transact_write_items(
ClientRequestToken=random_string(length=36),
TransactItems=transaction)
# 37 characters is too long for ClientRequestToken, a ValidationException
# results:
with pytest.raises(ClientError, match='ValidationException.*[cC]lientRequestToken'):
test_table_s.meta.client.transact_write_items(
ClientRequestToken=random_string(length=37),
TransactItems=transaction)
# Check that if the same ClientRequestToken is used for two transactions,
# but they are different, an IdempotentParameterMismatch error is thrown.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_clientrequesttoken_mismatch(test_table_s):
transaction1 = [{'Put': {'TableName': test_table_s.name, 'Item': {'p': random_string()}}}]
transaction2 = [{'Put': {'TableName': test_table_s.name, 'Item': {'p': random_string()}}}]
token = random_string()
test_table_s.meta.client.transact_write_items(
ClientRequestToken=token, TransactItems=transaction1)
with pytest.raises(ClientError, match='IdempotentParameterMismatchException'):
test_table_s.meta.client.transact_write_items(
ClientRequestToken=token, TransactItems=transaction2)
# All tests above involved a transaction with a single action. Here
# we finally begin to check trasactions with multiple actions. Let's
# begin with a successful TransactWriteItems transaction, where some of
# the actions have successful conditions, and some don't have conditions
# at all, and all the actions are performed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_multi_action_true(test_table_s):
p1 = random_string()
p2 = random_string()
p3 = random_string()
item1 = {'p': p1, 'x': 'dog'}
item2 = {'p': p2, 'x': 'cat'}
test_table_s.meta.client.transact_write_items(TransactItems=[
# unconditional Put
{ 'Put': {
'TableName': test_table_s.name,
'Item': item1
}},
# Put with true condition (attribute_not_exists(p) is true
# when the item doesn't exist).
{ 'Put': {
'TableName': test_table_s.name,
'Item': item2,
'ConditionExpression': 'attribute_not_exists(p)'
}},
# ConditionCheck with true condition. It is true, but doesn't
# cause anything to be written for key p3.
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p3},
'ConditionExpression': 'attribute_not_exists(p)'
}},
])
# p1 and p2 supposed to have been written by the transaction, but p3
# wasn't because it was only involved in a ConditionCheck.
assert item1 == test_table_s.get_item(Key={'p': p1}, ConsistentRead=True)['Item']
assert item2 == test_table_s.get_item(Key={'p': p2}, ConsistentRead=True)['Item']
assert 'Item' not in test_table_s.get_item(Key={'p': p3}, ConsistentRead=True)
# Test a transaction with several actions, one of which has a false
# condition. The entire transaction should fail, and none of its actions
# (not even those with true conditions or no conditions) should be performed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_multi_action_false(test_table_s):
p1 = random_string()
p2 = random_string()
p3 = random_string()
item1 = {'p': p1, 'x': 'dog'}
item2 = {'p': p2, 'x': 'cat'}
with pytest.raises(ClientError, match='TransactionCanceledException'):
test_table_s.meta.client.transact_write_items(TransactItems=[
# unconditional Put
{ 'Put': {
'TableName': test_table_s.name,
'Item': item1
}},
# Put with true condition (attribute_not_exists(p) is true
# when the item doesn't exist).
{ 'Put': {
'TableName': test_table_s.name,
'Item': item2,
'ConditionExpression': 'attribute_not_exists(p)'
}},
# ConditionCheck with a *false* condition. It should cause the
# entire transaction to be rejected.
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p3},
'ConditionExpression': 'attribute_exists(p)'
}},
])
# Because the entire transaction was rejected, none of its actions
# should have been done. Items for p1, p2 and p3 should all not exist.
assert 'Item' not in test_table_s.get_item(Key={'p': p1}, ConsistentRead=True)
assert 'Item' not in test_table_s.get_item(Key={'p': p2}, ConsistentRead=True)
assert 'Item' not in test_table_s.get_item(Key={'p': p3}, ConsistentRead=True)
# Test that it's not allowed for two actions in the same transaction to
# target the same item (this limitation also includes ConditionCheck
# actions).
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_multi_action_conflict(test_table_s):
p1 = random_string()
p2 = random_string()
with pytest.raises(ClientError, match='ValidationException.*one item'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}},
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}}])
with pytest.raises(ClientError, match='ValidationException.*one item'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}},
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p1},
'ConditionExpression': 'attribute_not_exists(p)'
}}])
with pytest.raises(ClientError, match='ValidationException.*one item'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}},
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p2},
'ConditionExpression': 'attribute_not_exists(p)'
}},
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p2},
'ConditionExpression': 'attribute_not_exists(p)'
}}])
# Test that a transaction may involve more than one table, not just more than
# one item.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_multi_table_true(test_table_s, test_table_ss):
p1 = random_string()
p2 = random_string()
c2 = random_string()
item1 = {'p': p1, 'x': 'dog'}
item2 = {'p': p2, 'c': c2, 'x': 'cat'}
test_table_s.meta.client.transact_write_items(TransactItems=[
# unconditional Put on the first table
{ 'Put': {
'TableName': test_table_s.name,
'Item': item1
}},
# Put with true condition on the second table
{ 'Put': {
'TableName': test_table_ss.name,
'Item': item2,
'ConditionExpression': 'attribute_not_exists(p)'
}}
])
# p1 and p2 supposed to have been written by the transaction on the
# two different tables
assert item1 == test_table_s.get_item(Key={'p': p1}, ConsistentRead=True)['Item']
assert item2 == test_table_ss.get_item(Key={'p': p2, 'c': c2}, ConsistentRead=True)['Item']
# Check that a TransactWriteItems with zero items is not allowed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_empty(test_table_s):
with pytest.raises(ClientError, match='ValidationException.*[tT]ransactItems'):
test_table_s.meta.client.transact_write_items(TransactItems=[])
# Check that a TransactWriteItems with 100 items is allowed. The limit
# used to be just 25 items, but it was increased to 100 in September 2022.
# The next test will check that *more* than 100 items is not allowed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_100(test_table_s):
p = random_string()
items = [{'p': p + str(i), 'x': i} for i in range(100)]
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': item,
}} for item in items])
# verify that all 100 items went in
for item in items:
assert item == test_table_s.get_item(Key={'p': item['p']}, ConsistentRead=True)['Item']
# Check that a transaction with 101 (>100) items is rejected.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_101(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*[tT]ransactItems.*100'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p + str(i)},
}} for i in range(101)])
# Beyond limiting a TransactWriteItems to 100 items (verified in tests above)
# DynamoDB has an aditional limit that "the aggregate size of the items in the
# transaction cannot exceed 4 MB". We can't write 5 MB in a transaction even
# if we build it from relatively small items (each not exceeding the 400 KB
# limit the size of an individual item size).
# Note that this test checks only Put actions which include the entire
# new item in the transaction - so the transaction itself reaches 5MB in
# size. In the next test we will check what happens for Update or Delete
# actions which may themselves have small size but refer to large items.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_put_5MB(test_table_s):
p = random_string()
# We will write 50 items of roughly 100KB each, reaching around 5MB
long = 'x'*100000
items = [{'p': p + str(i), 'x': long} for i in range(50)]
with pytest.raises(ClientError, match='ValidationException.*size cannot exceed'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': item,
}} for item in items])
# In the previous test (test_transact_write_items_put_5MB) we saw that we
# can't have over 4 MB of new data in a transaction. But the DynamoDB
# TransactWriteItems documentation is ambiguous on what this 4 MB limit
# applies to: The documentation says that it's not allowed that "the aggregate
# size of the items in the transaction exceeds 4 MB". Is this the size of the
# items *mentioned* in the transaction, or the size of items *in* the
# transaction? In other words, if we have a transaction doing Deletes of
# large items - where the transaction itself is very small but the total item
# size exceeds 4 MB - is this allowed or not?
# It turns out that the answer is that the size of the transaction request
# (DynamoDB refers to it as "transaction payload" in the error messages) is
# what matters, not the size of the items. A delete transaction that deletes
# 5 MB of items but the transaction itself is small - is allowed.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_delete_5MB(test_table_s):
p = random_string()
# Write (not in a transaction) 50 items of roughly 100 KB each, so their
# total size is roughly 5 MB.
long = 'x'*100000
items = [{'p': p + str(i), 'x': long} for i in range(50)]
with test_table_s.batch_writer() as batch:
for item in items:
batch.put_item(item)
# Now try to delete those 50 items in one transaction. The transaction
# request is very small, but the total size of the items to be deleted
# is around 5 MB. This transaction *is* allowed showing that DynamoDB's
# limit is really on the transaction size, not the "aggregate size of the
# items" mentioned in the documentation.
try:
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Delete': {
'TableName': test_table_s.name,
'Key': {'p': item['p']},
}} for item in items])
# Verify the deletes worked
for item in items:
assert 'Item' not in test_table_s.get_item(Key={'p': item['p']}, ConsistentRead=True)
except ClientError as e:
# A 5 MB write transaction takes roughly 10,000 WCUs, which DynamoDB
# sometimes refuses to do on a brand-new on-demand table, so the
# transaction gets rejected. But if this happens, it's an expected
# failure. It's not a real failure (a real failure would have been
# the transaction failing on something other than ThrottlingException).
if e.response['Error']['Code'] == 'ThrottlingException':
pytest.xfail()
else:
raise
# But verify that aggregate transaction size of 3MB is fine. Note that
# individual items are limited to 400KB, so we must a transaction with
# several smaller items.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_put_3MB(test_table_s):
p = random_string()
# We will write 30 items of roughly 100KB each, reaching around 3MB
long = 'x'*100000
items = [{'p': p + str(i), 'x': long} for i in range(30)]
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': item,
}} for item in items])
# verify that all 30 items went in
for item in items:
assert item == test_table_s.get_item(Key={'p': item['p']}, ConsistentRead=True)['Item']
# In various tests above we checked that failed conditions result in a
# TransactionCanceledException. But this error comes with CancellationReasons
# and in the following test we want to test these.
# Note that this test only checks the cancellation reasons "None" and
# "ConditionalCheckFailed". The test after it will check the "ValidationError"
# reason. There are additional reasons, but these tests are not designed to
# be able to reach those cases:
# 1. ItemCollectionSizeLimitExceeded - when there is an LSI and the write
# caused the item collection (Scylla partition) to go over 10 GB.
# 2. TransactionConflict - when there is a conflict with another transaction.
# As explained above, we don't test concurrent transactions in this test
# suite.
# 3. ProvisionedThroughputExceeded - self-explanatory, not tested in these
# tests.
# 4. ThrottlingError - similar, for on-demand tables that haven't scaled
# enough yet.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_cancellation_reasons_conditionalcheckfailed(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
# The condition attribute_exists(p) is false for an
# item that doesn't exist
'ConditionExpression': 'attribute_exists(p)'
}}])
# The error object should have a CancellationReasons member. Our
# transaction has one item, and one failure, so the CancellationReasons
# should be a one-member array. This member has a code, and a message.
assert 'CancellationReasons' in e.value.response
reasons = e.value.response['CancellationReasons']
assert len(reasons) == 1
assert reasons[0]['Code'] == 'ConditionalCheckFailed'
# The Message is a user-readable message like "The conditional request
# failed". We'll assert it exists, but not insist what the text is
assert 'Message' in reasons[0]
# Now do the same thing with a transaction with multiple actions, some
# with failed conditions so the entire transaction gets canceled.
# We'll see that the CancellationReasons lists each one of the actions
# in order, and says which ones had a failed condition.
p1 = random_string()
p2 = random_string()
p3 = random_string()
p4 = random_string()
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[
# Unconditional Put
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p1}
}},
# Put with true condition (attribute_not_exists(p) is true
# when the item doesn't exist).
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p2},
'ConditionExpression': 'attribute_not_exists(p)'
}},
# ConditionCheck with a *false* condition. It should cause the
# entire transaction to be rejected.
{ 'ConditionCheck': {
'TableName': test_table_s.name,
'Key': {'p': p3},
'ConditionExpression': 'attribute_exists(p)'
}},
# Another false condition, on a Put action:
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p4},
'ConditionExpression': 'attribute_exists(p)'
}},
])
assert 'CancellationReasons' in e.value.response
reasons = e.value.response['CancellationReasons']
# The CancellationReasons are listed in the same order of the actions
# in the transaction - we expect the first two actions to have not
# failed, so they have Code="None", and the third and forth both failed.
# It's interesting that DynamoDB does report both failures - and doesn't
# "short circuit" after the first one.
assert len(reasons) == 4
assert reasons[0]['Code'] == 'None'
assert reasons[1]['Code'] == 'None'
assert reasons[2]['Code'] == 'ConditionalCheckFailed'
assert reasons[3]['Code'] == 'ConditionalCheckFailed'
# Actions "None" reason don't come with a user-readable message,
# but others do.
assert 'Message' not in reasons[0]
assert 'Message' not in reasons[1]
assert 'Message' in reasons[2]
assert 'Message' in reasons[3]
# Another possible cancellation reason is a ValidationError in one of the
# actions - for example an attempt to write a wrong type into a key or
# exceeding the size limit on a key or the size of an item, or exceeding the
# nesting level limits.
# Note that the actions are validated before any of them actually run,
# so if any action has a ValidationError, other correctly-validated
# actions will return a "None" reason - even if potentially they should
# have been a false condition.
# In fact it turns out (and we'll confirm in this test) that if two actions
# have a validation error, only the first error is detected, and the server
# doesn't continue to find the validation error in the second action. It's
# probably not important that Alternator be compatible with this behavior,
# but if we can, why not.
#
# Surprisingly, many validations actually return a ValidationException
# on the entire transaction instead of a TransactionCanceledException with
# one ValidationError in its cancellation reason. For example (we have a
# test for this below), a reference to a missing ExpressionAttributeNames.
# It's not clear how DynamoDB decided which case will return a
# ValidationException and which a ValidationError, but let's be compatible
# with what DynamoDB does.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_cancellation_reasons_validationerror(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
# The key p should be a string, not a number, so this
# item has a validation error.
'Item': {'p': 7}
}},
{ 'Put': {
'TableName': test_table_s.name,
# A second validation error. As explained above, as soon
# as the first validation error was found above, this action
# will not be reached and its validation error will not be
# detected. So we expect "None" reason on this action.
'Item': {'p': 8}
}},
{ 'Put': {
'TableName': test_table_s.name,
# This condition is false (the item does *not* exist yet),
# and would have failed the tranaction, but the server won't
# even get around to checking that because of the validation
# error in the other actions. So we will expect to get a
# "None" reason on this action.
'ConditionExpression': 'attribute_exists(p)',
'Item': {'p': p}
}},
])
assert 'CancellationReasons' in e.value.response
reasons = e.value.response['CancellationReasons']
assert len(reasons) == 3
assert reasons[0]['Code'] == 'ValidationError'
assert reasons[1]['Code'] == 'None'
assert reasons[2]['Code'] == 'None'
assert 'Message' in reasons[0]
# "None" reason doesn't come with a message:
assert 'Message' not in reasons[1]
assert 'Message' not in reasons[2]
# Verify that even if there is just *one* action in the transaction,
# on certain types of errors like the incorrect key type we used above
# in test_transact_write_cancellation_reasons_validationerror will get a
# TransactionCanceledException with a single ValidationError - not a
# ValidationException. Below we'll see in other tests that this is not
# always true - in some other types of errors, we actually do get a
# ValidationException.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_cancellation_reasons_validationerror_one(test_table_s):
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
# The key p should be a string, not a number, so this
# item has a validation error.
'Item': {'p': 7}
}}])
reasons = e.value.response['CancellationReasons']
assert reasons[0]['Code'] == 'ValidationError'
assert 'Message' in reasons[0]
# Check the ReturnValuesOnConditionCheckFailure parameter, which allows
# the user to retrieve the content of the item that caused a specific
# condition to fail.
# Note that ReturnValuesOnConditionCheckFailure is specified for a
# specific action, not for the entire transaction.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_returnvaluesonconditioncheckfailure(test_table_s):
p = random_string()
item = {'p': p, 'x': 42, 'y': 'dog'}
test_table_s.put_item(Item=item)
# Check ReturnValuesOnConditionCheckFailure="ALL_OLD"
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'ReturnValuesOnConditionCheckFailure': 'ALL_OLD',
'TableName': test_table_s.name,
'Key': {'p': p},
# x is really 42, not 43, so this condition is false
'ConditionExpression': 'x = :fourtythree',
'UpdateExpression': 'SET x = x + :one',
'ExpressionAttributeValues': {':one': 1, ':fourtythree': 43},
}}])
# The relevant CancellationReasons will now have an "Item".
# The returned reasons[0]['Item'] has DynamoDB JSON encoding, and in
# this specific place (inside the error object) boto3 forgot to
# deserialize the JSON encoding for us (like it does automatically
# in other place) - so we need to do this deserialization manually
# with TypeDeserializer before we can compare it to the expected "item".
assert 'CancellationReasons' in e.value.response
reasons = e.value.response['CancellationReasons']
assert len(reasons) == 1
assert reasons[0]['Code'] == 'ConditionalCheckFailed'
assert 'Message' in reasons[0]
deserializer = TypeDeserializer()
got_item = {x:deserializer.deserialize(y) for (x,y) in reasons[0]['Item'].items()}
assert item == got_item
# The same transaction with ReturnValuesOnConditionCheckFailure="NONE"
# doesn't return the Item:
with pytest.raises(ClientError, match='TransactionCanceledException') as e:
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'ReturnValuesOnConditionCheckFailure': 'NONE',
'TableName': test_table_s.name,
'Key': {'p': p},
'ConditionExpression': 'x = :fourtythree',
'UpdateExpression': 'SET x = x + :one',
'ExpressionAttributeValues': {':one': 1, ':fourtythree': 43},
}}])
reasons = e.value.response['CancellationReasons']
assert len(reasons) == 1
assert reasons[0]['Code'] == 'ConditionalCheckFailed'
assert 'Item' not in reasons[0]
# Setting ReturnValuesOnConditionCheckFailure to anything else, e.g.,
# lowercase 'none', or 'dog', is a validation error. Surprisingly,
# even though ReturnValuesOnConditionCheckFailure is per-action, and
# DynamoDB can return TransactionCanceledException with a per-action
# ValidationError, in this case it doesn't do that - it returns a
# ValidationException for the entire request.
for option in ['none', 'dog']:
with pytest.raises(ClientError, match='ValidationException.*[rR]eturnValuesOnConditionCheckFailure'):
test_table_s.meta.client.transact_write_items(TransactItems=[{
'Update': {
'ReturnValuesOnConditionCheckFailure': option,
'TableName': test_table_s.name,
'Key': {'p': p},
'ConditionExpression': 'x = :fourtythree',
'UpdateExpression': 'SET x = x + :one',
'ExpressionAttributeValues': {':one': 1, ':fourtythree': 43},
}}])
# Various checks for ExpressionAttributeValues and ExpressionAttributeNames
# in a ConditionExpression, such as missing or unused entries in those arrays.
# We already have tests for ConditionExpression in test_condition_expression.py
# but here the problem happens inside one action instead of the whole
# transaction so we need to check if it's also recognized, and whether it
# causes a TransactionCanceledException with one ValidationError or a
# ValidationException on the entire transaction. It turns out that it's
# the latter.
# Check ConditionExpression reference to missing ExpressionAttributeNames
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_missing_name(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*#xyz'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
# The reference "#xyz" is missing in ExpressionAttributeNames
'ConditionExpression': 'attribute_exists(#xyz)'
}}])
# Check ConditionExpression value missing in ExpressionAttributeValues
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_missing_value(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*:xyz'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
# The reference ":xyz" is missing in ExpressionAttributeValues
'ConditionExpression': 'p = :xyz'
}}])
# Check unused name in ExpressionAttributeName
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_unused_name(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*unused.*#xyz'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
'ConditionExpression': 'attribute_exists(p)',
# The name "#xyz" isn't used in in any expression
'ExpressionAttributeNames': {'#xyz': 'x'}
}}])
# Check unused value in ExpressionAttributeValues
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_unused_value(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*unused.*:xyz'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
'ConditionExpression': 'attribute_exists(p)',
# The value ":xyz" isn't used in in any expression
'ExpressionAttributeValues': {':xyz': 7}
}}])
# Syntax error in a ConditionExpression in one action also returns a
# ValidationException for the entire transaction, not per item:
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_syntax_error(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*Syntax error'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
# "hello!" is a syntax error as an expression
'ConditionExpression': 'hello!',
}}])
# Some other errors in a ConditionExpression, like an ExpressionAttributeNames
# with an integer instead of a string (an attribute name), can generate a
# a SerializationException instead of a ValidationException. It's not clear
# to me why this is important, and I think it's fine if Alternator would
# return a ValidationException here and we change the test to accept both.
# But the important point is that the single exception is returned for the
# entire transaction - not per item.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_write_items_serialization_exception(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='SerializationException'):
test_table_s.meta.client.transact_write_items(TransactItems=[
{ 'Put': {
'TableName': test_table_s.name,
'Item': {'p': p},
'ConditionExpression': 'attribute_exists(p)',
# ExpressionAttributeNames expects strings (names of
# attributes), not the integer 7.
'ExpressionAttributeNames': {'#xyz': 7}
}}])
##########################################################################
# Tests for TransactGetItems functionality
##########################################################################
# Test basic transaction with one "Get" action
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_one(test_table_s):
p = random_string()
x = random_string()
item = {'p': p, 'x': x}
test_table_s.put_item(Item=item)
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
}}])
assert 'Responses' in ret
assert len(ret['Responses']) == 1
assert 'Item' in ret['Responses'][0]
assert item == ret['Responses'][0]['Item']
# If a Get transaction can't find an item with the given key, it's not
# an error - one of the Responses entries will just not have an "Item":
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_missing(test_table_s):
p = random_string()
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
}}])
assert 'Responses' in ret
assert len(ret['Responses']) == 1
assert 'Item' not in ret['Responses'][0]
# Test the ProjectionExpression parameter for a Get action, asking not to
# get the entire item and rather just get specific attributes. See more
# extensive tests for ProjectionExpression in test_projection_expression.py.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_projection_expression(test_table_s):
p = random_string()
item = {'p': p, 'x': 1, 'y': 2, 'z': 3}
test_table_s.put_item(Item=item)
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
'ProjectionExpression': 'x,z'
}}])
assert 'Responses' in ret
assert len(ret['Responses']) == 1
assert 'Item' in ret['Responses'][0]
assert {'x': item['x'], 'z': item['z']} == ret['Responses'][0]['Item']
# ProjectionExpression also supports ExpressionAttributeNames.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_projection_expression_attribute_names(test_table_s):
p = random_string()
item = {'p': p, 'x': 1, 'y': 2, 'z': 3}
test_table_s.put_item(Item=item)
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
'ProjectionExpression': '#xx,#zz',
'ExpressionAttributeNames': {'#xx': 'x', '#zz': 'z'}
}}])
assert 'Responses' in ret
assert len(ret['Responses']) == 1
assert 'Item' in ret['Responses'][0]
assert {'x': item['x'], 'z': item['z']} == ret['Responses'][0]['Item']
# If ExpressionAttributeNames is missing a name, or has an unused name,
# it's an error. As we saw above in other cases, it's a ValidationException
# for the entire transaction - not a TransactionCanceledException.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_unused_expressionattributenames(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*unused.*#qq'):
test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
'ProjectionExpression': '#xx,#zz',
# The reference "#qq" isn't used in the ProjectionExpression
'ExpressionAttributeNames': {'#xx': 'x', '#zz': 'z', '#qq': 'q'}
}}])
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_missing_expressionattributenames(test_table_s):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*#zz'):
test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': p},
'ProjectionExpression': '#xx,#zz',
# The reference "#zz" is missing in ExpressionAttributeNames
'ExpressionAttributeNames': {'#xx': 'x'}
}}])
# Test the ability to read multiple items in one transaction. We can actually
# read 100 small items in one transaction - the limit used to be just 25
# items, but it was increased to 100 in September 2022 so let's verify that
# it works.
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_100(test_table_s):
p = random_string()
items = [{'p': p + str(i), 'x': i} for i in range(100)]
with test_table_s.batch_writer() as batch:
for item in items:
batch.put_item(item)
ret = test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': item['p']},
}} for item in items])
# verify that all 100 items were read
for item in items:
assert item == test_table_s.get_item(Key={'p': item['p']}, ConsistentRead=True)['Item']
assert 'Responses' in ret
assert len(ret['Responses']) == 100
for i, response in enumerate(ret['Responses']):
assert 'Item' in response
assert response['Item'] == items[i]
# A transaction with 100 read actions is the limit, and 101 are not allowed:
@pytest.mark.xfail(reason="#5064 - transactions not yet supported")
def test_transact_get_items_101(test_table_s):
with pytest.raises(ClientError, match='ValidationException.*[tT]ransactItems.*100'):
test_table_s.meta.client.transact_get_items(TransactItems=[
{ 'Get': {
'TableName': test_table_s.name,
'Key': {'p': str(i)},
}} for i in range(101)])
Reference in New Issue View Git Blame Copy Permalink