mirrors/scylladb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
copilot/fix-task-api-docs-and-node-ops
scylladb/test/alternator/test_gsi.py
Nadav Har'El 5b78e1cebe test/alternator: tests for ExclusiveStartKey in GSI 
After in the previous patches we added more exhaustive testing for
the ExclusiveStartKey feature of Query and Scan, in this patch we
add tests for this feature in the context of GSIs.

Most interestingly, the ExclusiveStartKey when querying a GSI
isn't just the key of the GSI, but also includes the key columns
of the base - in other words, it is the key that Scylla uses for
its materialized view.

The tests here confirm that paging on GSI works - this paging uses
ExclusiveStartKey of course - but also what is the specific structure
and meaning of the content of ExclusiveStartKey.

We also include two xfailing tests which again, like in the previous
patches, show we don't do enough validation (issue #26988) and
don't recognize wrong values or spurious columns in ExclusiveStartKey.

As usual, all new tests pass on DynamoDB, and all except the xfailing
ones pass on Alternator.

Signed-off-by: Nadav Har'El <nyh@scylladb.com>
2025-11-17 22:07:28 +02:00

2366 lines
126 KiB
Python
Raw Permalink Blame History

# Copyright 2019-present ScyllaDB
#
# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
# Tests of GSI (Global Secondary Indexes)
#
# All the tests in this file create a GSI together with the base table - the
# separate file test_gsi_updatetable.py has tests for the ability to use
# UpdateTable to add or remove GSIs on existing tables.
#
# Note that many of these tests are slower than usual, because many of them
# need to create new tables and/or new GSIs of different types, operations
# which are extremely slow in DynamoDB, often taking minutes (!).
import pytest
import time
import itertools
from botocore.exceptions import ClientError
from .util import create_test_table, random_string, random_bytes, full_scan, full_query, multiset, list_tables, new_test_table, wait_for_gsi, unique_table_name
# GSIs only support eventually consistent reads, so tests that involve
# writing to a table and then expect to read something from it cannot be
# guaranteed to succeed without retrying the read. The following utility
# functions make it easy to write such tests.
# Note that in practice, there repeated reads are almost never necessary:
# Amazon claims that "Changes to the table data are propagated to the global
# secondary indexes within a fraction of a second, under normal conditions"
# and indeed, in practice, the tests here almost always succeed without a
# retry.
# However, it is worthwhile to differentiate between the case where the
# result set is not *yet* complete (which is ok, and requires retry), and
# the case that the result set has wrong data. In the latter case, the
# test will surely fail and no amount of retry will help, so we should
# fail quickly, to avoid xfailing tests being very slow.
def assert_index_query(table, index_name, expected_items, **kwargs):
expected = multiset(expected_items)
for i in range(5):
got = multiset(full_query(table, IndexName=index_name, ConsistentRead=False, **kwargs))
if expected == got:
return
elif got - expected:
# If we got any items that weren't expected, there's no point to retry.
pytest.fail("assert_index_query() found unexpected items: " + str(got - expected))
print('assert_index_query retrying')
time.sleep(1)
assert multiset(expected_items) == multiset(full_query(table, IndexName=index_name, ConsistentRead=False, **kwargs))
def assert_index_scan(table, index_name, expected_items, **kwargs):
expected = multiset(expected_items)
for i in range(5):
got = multiset(full_scan(table, IndexName=index_name, ConsistentRead=False, **kwargs))
if expected == got:
return
elif got - expected:
# If we got any items that weren't expected, there's no point to retry.
pytest.fail("assert_index_scan() found unexpected items: " + str(got - expected))
print('assert_index_scan retrying')
time.sleep(1)
assert multiset(expected_items) == multiset(full_scan(table, IndexName=index_name, ConsistentRead=False, **kwargs))
# Although quite silly, it is actually allowed to create an index which is
# identical to the base table.
# The following test does not work for KA/LA tables due to #6157, but we
# no longer allow writing those in Scylla.
def test_gsi_identical(dynamodb):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }],
AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [{ 'AttributeName': 'p', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
}
]) as table:
items = [{'p': random_string(), 'x': random_string()} for i in range(10)]
with table.batch_writer() as batch:
for item in items:
batch.put_item(item)
# Scanning the entire table directly or via the index yields the same
# results (in different order).
assert multiset(items) == multiset(full_scan(table))
assert_index_scan(table, 'hello', items)
# We can't scan a non-existent index
with pytest.raises(ClientError, match='ValidationException'):
full_scan(table, ConsistentRead=False, IndexName='wrong')
# One of the simplest forms of a non-trivial GSI: The base table has a hash
# and sort key, and the index reverses those roles. Other attributes are just
# copied.
@pytest.fixture(scope="module")
def test_table_gsi_1(dynamodb):
table = create_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'c', 'KeyType': 'RANGE' }
],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'c', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'c', 'KeyType': 'HASH' },
{ 'AttributeName': 'p', 'KeyType': 'RANGE' },
],
'Projection': { 'ProjectionType': 'ALL' }
}
],
)
yield table
table.delete()
def test_gsi_simple(test_table_gsi_1):
items = [{'p': random_string(), 'c': random_string(), 'x': random_string()} for i in range(10)]
with test_table_gsi_1.batch_writer() as batch:
for item in items:
batch.put_item(item)
c = items[0]['c']
# The index allows a query on just a specific sort key, which isn't
# allowed on the base table.
with pytest.raises(ClientError, match='ValidationException'):
full_query(test_table_gsi_1, KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
expected_items = [x for x in items if x['c'] == c]
assert_index_query(test_table_gsi_1, 'hello', expected_items,
KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
# Scanning the entire table directly or via the index yields the same
# results (in different order).
assert_index_scan(test_table_gsi_1, 'hello', full_scan(test_table_gsi_1))
def test_gsi_same_key(test_table_gsi_1):
c = random_string();
# All these items have the same sort key 'c' but different hash key 'p'
items = [{'p': random_string(), 'c': c, 'x': random_string()} for i in range(10)]
with test_table_gsi_1.batch_writer() as batch:
for item in items:
batch.put_item(item)
assert_index_query(test_table_gsi_1, 'hello', items,
KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
# Check we get an appropriate error when trying to read a non-existing index
# of an existing table. Although the documentation specifies that a
# ResourceNotFoundException should be returned if "The operation tried to
# access a nonexistent table or index", in fact in the specific case that
# the table does exist but an index does not - we get a ValidationException.
def test_gsi_missing_index(test_table_gsi_1):
with pytest.raises(ClientError, match='ValidationException.*wrong_name'):
full_query(test_table_gsi_1, IndexName='wrong_name',
KeyConditions={'x': {'AttributeValueList': [1], 'ComparisonOperator': 'EQ'}})
with pytest.raises(ClientError, match='ValidationException.*wrong_name'):
full_scan(test_table_gsi_1, IndexName='wrong_name')
# Nevertheless, if the table itself does not exist, a query should return
# a ResourceNotFoundException, not ValidationException:
def test_gsi_missing_table(dynamodb):
with pytest.raises(ClientError, match='ResourceNotFoundException'):
dynamodb.meta.client.query(TableName='nonexistent_table', IndexName='any_name', KeyConditions={'x': {'AttributeValueList': [1], 'ComparisonOperator': 'EQ'}})
with pytest.raises(ClientError, match='ResourceNotFoundException'):
dynamodb.meta.client.scan(TableName='nonexistent_table', IndexName='any_name')
# Verify that strongly-consistent reads on GSI are *not* allowed.
def test_gsi_strong_consistency(test_table_gsi_1):
with pytest.raises(ClientError, match='ValidationException.*Consistent'):
full_query(test_table_gsi_1, KeyConditions={'c': {'AttributeValueList': ['hi'], 'ComparisonOperator': 'EQ'}}, IndexName='hello', ConsistentRead=True)
with pytest.raises(ClientError, match='ValidationException.*Consistent'):
full_scan(test_table_gsi_1, IndexName='hello', ConsistentRead=True)
# Test that setting an indexed string column to an empty string is illegal,
# since keys cannot contain empty strings
# Test this in the different write operations - PutItem, UpdateItem and
# BatchWriteItem, to verify we didn't miss the checks in any of those
# code paths.
def test_gsi_empty_value(test_table_gsi_2):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*empty'):
test_table_gsi_2.put_item(Item={'p': p, 'x': ''})
with pytest.raises(ClientError, match='ValidationException.*empty'):
test_table_gsi_2.update_item(Key={'p': p}, AttributeUpdates={'x': {'Value': '', 'Action': 'PUT'}})
with pytest.raises(ClientError, match='ValidationException.*empty'):
with test_table_gsi_2.batch_writer() as batch:
batch.put_item({'p': p, 'x': ''})
def test_gsi_empty_value_with_range_key(test_table_gsi_3):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*empty'):
test_table_gsi_3.put_item(Item={'p': p, 'a': '', 'b': p})
with pytest.raises(ClientError, match='ValidationException.*empty'):
test_table_gsi_3.put_item(Item={'p': p, 'a': p, 'b': ''})
with pytest.raises(ClientError, match='ValidationException.*empty'):
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': '', 'Action': 'PUT'}, 'b': {'Value': p, 'Action': 'PUT'}})
with pytest.raises(ClientError, match='ValidationException.*empty'):
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': p, 'Action': 'PUT'}, 'b': {'Value': '', 'Action': 'PUT'}})
with pytest.raises(ClientError, match='ValidationException.*empty'):
with test_table_gsi_3.batch_writer() as batch:
batch.put_item({'p': p, 'a': '', 'b': p})
with pytest.raises(ClientError, match='ValidationException.*empty'):
with test_table_gsi_3.batch_writer() as batch:
batch.put_item({'p': p, 'a': p, 'b': ''})
# In test_gsi_empty_value we validated that writing an empty string to
# column that is a GSI key fails. We also checked BatchWriteItem. Let's
# verify that with BatchWriteItem, if one of the writes fail this
# verification, none of the other writes in the batch get done either.
def test_gsi_empty_value_in_bigger_batch_write(test_table_gsi_2):
p1 = random_string()
p2 = random_string()
p3 = random_string()
items = [{'p': p1, 'x': p1}, {'p': p2, 'x': ''}, {'p': p3, 'x': p3}]
with pytest.raises(ClientError, match='ValidationException.*empty'):
with test_table_gsi_2.batch_writer() as batch:
for item in items:
batch.put_item(item)
for p in [p1, p2, p3]:
assert not 'Item' in test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)
# Dynamodb supports special way of setting NULL value.
# It's different than non existing value.
def test_gsi_null_value(test_table_gsi_2):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*NULL'):
test_table_gsi_2.put_item(Item={'p': p, 'x': None})
with pytest.raises(ClientError, match='ValidationException.*NULL'):
test_table_gsi_2.update_item(Key={'p': p}, AttributeUpdates={'x': {'Value': None, 'Action': 'PUT'}})
with pytest.raises(ClientError, match='ValidationException.*NULL'):
with test_table_gsi_2.batch_writer() as batch:
batch.put_item({'p': p, 'x': None})
def test_gsi_null_value_with_range_key(test_table_gsi_3):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*NULL'):
test_table_gsi_3.put_item(Item={'p': p, 'a': None, 'b': p})
with pytest.raises(ClientError, match='ValidationException.*NULL'):
test_table_gsi_3.put_item(Item={'p': p, 'a': p, 'b': None})
with pytest.raises(ClientError, match='ValidationException.*NULL'):
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': None, 'Action': 'PUT'}, 'b': {'Value': p, 'Action': 'PUT'}})
with pytest.raises(ClientError, match='ValidationException.*NULL'):
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': p, 'Action': 'PUT'}, 'b': {'Value': None, 'Action': 'PUT'}})
with pytest.raises(ClientError, match='ValidationException.*NULL'):
with test_table_gsi_3.batch_writer() as batch:
batch.put_item({'p': p, 'a': None, 'b': p})
with pytest.raises(ClientError, match='ValidationException.*NULL'):
with test_table_gsi_3.batch_writer() as batch:
batch.put_item({'p': p, 'a': p, 'b': None})
# Verify that a GSI is correctly listed in describe_table
def test_gsi_describe(test_table_gsi_1):
desc = test_table_gsi_1.meta.client.describe_table(TableName=test_table_gsi_1.name)
assert 'Table' in desc
assert 'GlobalSecondaryIndexes' in desc['Table']
gsis = desc['Table']['GlobalSecondaryIndexes']
assert len(gsis) == 1
gsi = gsis[0]
assert gsi['IndexName'] == 'hello'
assert gsi['Projection'] == {'ProjectionType': 'ALL'}
assert gsi['KeySchema'] == [{'KeyType': 'HASH', 'AttributeName': 'c'},
{'KeyType': 'RANGE', 'AttributeName': 'p'}]
# The index's ARN should look like the table's ARN followed by /index/<indexname>.
assert gsi['IndexArn'] == desc['Table']['TableArn'] + '/index/hello'
# TODO: check also ProvisionedThroughput
# Test that an already-existing GSI should be listed by DescribeTable with
# IndexStatus=ACTIVE. A GSI that was just created with UpdateTable and being
# backfilled might be in other states, but that case is tested in different
# tests in test_gsi_updatetable.py.
# Reproduces #11471.
def test_gsi_describe_indexstatus(test_table_gsi_1):
# In DynamoDB, a GSI created together with the table is always immediately
# ACTIVE, but this is not always true in Alternator: Although a new table
# is completely empty and its "view building" phase has nothing to do,
# this "nothing" can still take a short while (especially in debug builds)
# and in the mean time the test might see the CREATING state and be flaky.
# So let's wait_for_gsi() just to be sure the view building is over.
# Note that this makes the explicit IndexStatus check below redundant,
# because wait_for_gsi() already does it..
wait_for_gsi(test_table_gsi_1, 'hello')
desc = test_table_gsi_1.meta.client.describe_table(TableName=test_table_gsi_1.name)
gsis = desc['Table']['GlobalSecondaryIndexes']
assert len(gsis) == 1
gsi = gsis[0]
assert 'IndexStatus' in gsi
assert gsi['IndexStatus'] == 'ACTIVE'
# In addition to the basic listing of an GSI in DescribeTable tested above,
# in this test we check additional fields that should appear in each GSI's
# description.
@pytest.mark.xfail(reason="issues #7550, #11466")
def test_gsi_describe_fields(test_table_gsi_1):
desc = test_table_gsi_1.meta.client.describe_table(TableName=test_table_gsi_1.name)
gsis = desc['Table']['GlobalSecondaryIndexes']
assert len(gsis) == 1
gsi = gsis[0]
assert 'IndexSizeBytes' in gsi # actual size depends on content
assert 'ItemCount' in gsi
# When a GSI's key includes an attribute not in the base table's key, we
# need to remember to add its type to AttributeDefinitions.
def test_gsi_missing_attribute_definition(dynamodb):
with pytest.raises(ClientError, match='ValidationException.*AttributeDefinitions'):
create_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[ { 'AttributeName': 'p', 'AttributeType': 'S' } ],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [ { 'AttributeName': 'c', 'KeyType': 'HASH' } ],
'Projection': { 'ProjectionType': 'ALL' }
}
])
# test_table_gsi_1_hash_only is a variant of test_table_gsi_1: It's another
# case where the index doesn't involve non-key attributes. Again the base
# table has a hash and sort key, but in this case the index has *only* a
# hash key (which is the base's hash key). In the materialized-view-based
# implementation, we need to remember the other part of the base key as a
# clustering key.
@pytest.fixture(scope="module")
def test_table_gsi_1_hash_only(dynamodb):
table = create_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'c', 'KeyType': 'RANGE' }
],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'c', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'c', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'ALL' }
}
],
)
yield table
table.delete()
def test_gsi_key_not_in_index(test_table_gsi_1_hash_only):
# Test with items with different 'c' values:
items = [{'p': random_string(), 'c': random_string(), 'x': random_string()} for i in range(10)]
with test_table_gsi_1_hash_only.batch_writer() as batch:
for item in items:
batch.put_item(item)
c = items[0]['c']
expected_items = [x for x in items if x['c'] == c]
assert_index_query(test_table_gsi_1_hash_only, 'hello', expected_items,
KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
# Test items with the same sort key 'c' but different hash key 'p'
c = random_string();
items = [{'p': random_string(), 'c': c, 'x': random_string()} for i in range(10)]
with test_table_gsi_1_hash_only.batch_writer() as batch:
for item in items:
batch.put_item(item)
assert_index_query(test_table_gsi_1_hash_only, 'hello', items,
KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
# Scanning the entire table directly or via the index yields the same
# results (in different order).
assert_index_scan(test_table_gsi_1_hash_only, 'hello', full_scan(test_table_gsi_1_hash_only))
# A second scenario of GSI. Base table has just hash key, Index has a
# different hash key - one of the non-key attributes from the base table.
@pytest.fixture(scope="module")
def test_table_gsi_2(dynamodb):
table = create_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'x', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'ALL' }
}
])
yield table
table.delete()
def test_gsi_2(test_table_gsi_2):
items1 = [{'p': random_string(), 'x': random_string()} for i in range(10)]
x1 = items1[0]['x']
x2 = random_string()
items2 = [{'p': random_string(), 'x': x2} for i in range(10)]
items = items1 + items2
with test_table_gsi_2.batch_writer() as batch:
for item in items:
batch.put_item(item)
expected_items = [i for i in items if i['x'] == x1]
assert_index_query(test_table_gsi_2, 'hello', expected_items,
KeyConditions={'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
expected_items = [i for i in items if i['x'] == x2]
assert_index_query(test_table_gsi_2, 'hello', expected_items,
KeyConditions={'x': {'AttributeValueList': [x2], 'ComparisonOperator': 'EQ'}})
# The previous tests just need to create rows in the materialized view.
# This test adds more elaborate operations which need to create new rows,
# modify existing rows, and delete rows of the materialized view.
# We use the schema test_table_gsi_2, and create, modify and delete the
# attribute "x" (a regular attribute in the base table, a partition key in
# the GSI) to cause all these different operations on the view rows, and
# check various code paths in the view update code.
def test_update_gsi_pk(test_table_gsi_2):
p = random_string()
x1 = random_string()
y = random_string()
z = random_string()
# Create a new GSI row (x1), see that it appears
test_table_gsi_2.put_item(Item={'p': p, 'x': x1, 'y': y, 'z': z})
assert_index_query(test_table_gsi_2, 'hello', [{'p': p, 'x': x1, 'y': y, 'z': z}],
KeyConditions={'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
# Update only the unrelated attribute y. Should leave the same row in
# the GSI (x=x1), just with a modified y (and unmodified z)
y = random_string()
test_table_gsi_2.update_item(Key={'p': p}, AttributeUpdates={'y': {'Value': y, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_2, 'hello', [{'p': p, 'x': x1, 'y': y, 'z': z}],
KeyConditions={'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
# Update the GSI's key attribute x to x2. The old row (x=x1) should
# disappear from the GSI, and the new row (x=x2) should appear, with the
# base row's "y" and "z" value that weren't changed in this update.
x2 = random_string()
test_table_gsi_2.update_item(Key={'p': p}, AttributeUpdates={'x': {'Value': x2, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_2, 'hello', [],
KeyConditions={'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_2, 'hello', [{'p': p, 'x': x2, 'y': y, 'z': z}],
KeyConditions={'x': {'AttributeValueList': [x2], 'ComparisonOperator': 'EQ'}})
# Delete only the attribute x from our base-table row. The row should
# remain in the table (with no x), but disappear from the view
test_table_gsi_2.update_item(Key={'p': p}, AttributeUpdates={'x': {'Action': 'DELETE'}})
assert test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'y': y, 'z': z}
assert_index_query(test_table_gsi_2, 'hello', [],
KeyConditions={'x': {'AttributeValueList': [x2], 'ComparisonOperator': 'EQ'}})
# Set x again to x1, see the view item re-appears in the view:
test_table_gsi_2.update_item(Key={'p': p}, AttributeUpdates={'x': {'Value': x1, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_2, 'hello', [{'p': p, 'x': x1, 'y': y, 'z': z}],
KeyConditions={'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
# Delete the entire item in the base table, the view row should also
# disappear
test_table_gsi_2.delete_item(Key={'p': p})
assert_index_query(test_table_gsi_2, 'hello', [],
KeyConditions={'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
# Similar to the previous test (test_update_gsi_pk) except that the base's
# non-key attribute is here a clustering key (sort key).
def test_update_gsi_ck(test_table_gsi_5):
p = random_string()
c = random_string()
x1 = random_string()
y = random_string()
z = random_string()
# Create a new GSI row (x1), see that it appears
test_table_gsi_5.put_item(Item={'p': p, 'c': c, 'x': x1, 'y': y, 'z': z})
assert_index_query(test_table_gsi_5, 'hello', [{'p': p, 'c': c, 'x': x1, 'y': y, 'z': z}],
KeyConditions={
'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
# Update only the unrelated attribute y. Should leave the same row in
# the GSI (x=x1), just with a modified y (and unmodified z)
y = random_string()
test_table_gsi_5.update_item(Key={'p': p, 'c': c}, AttributeUpdates={'y': {'Value': y, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_5, 'hello', [{'p': p, 'c': c, 'x': x1, 'y': y, 'z': z}],
KeyConditions={
'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
# Update the GSI's key attribute x to x2. The old row (x=x1) should
# disappear from the GSI, and the new row (x=x2) should appear, with the
# base row's "y" and "z" value that weren't changed in this update.
x2 = random_string()
test_table_gsi_5.update_item(Key={'p': p, 'c': c}, AttributeUpdates={'x': {'Value': x2, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_5, 'hello', [],
KeyConditions={
'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_5, 'hello', [{'p': p, 'c': c, 'x': x2, 'y': y, 'z': z}],
KeyConditions={
'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x2], 'ComparisonOperator': 'EQ'}})
# Delete only the attribute x from our base-table row. The row should
# remain in the table (with no x), but disappear from the view
test_table_gsi_5.update_item(Key={'p': p, 'c': c}, AttributeUpdates={'x': {'Action': 'DELETE'}})
assert test_table_gsi_5.get_item(Key={'p': p, 'c': c}, ConsistentRead=True)['Item'] == {'p': p, 'c': c, 'y': y, 'z': z}
assert_index_query(test_table_gsi_5, 'hello', [],
KeyConditions={
'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x2], 'ComparisonOperator': 'EQ'}})
# Set x again to x1, see the view item re-appears in the view:
test_table_gsi_5.update_item(Key={'p': p, 'c': c}, AttributeUpdates={'x': {'Value': x1, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_5, 'hello', [{'p': p, 'c': c, 'x': x1, 'y': y, 'z': z}],
KeyConditions={
'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
# Delete the entire item in the base table, the view row should also
# disappear
test_table_gsi_5.delete_item(Key={'p': p, 'c': c})
assert_index_query(test_table_gsi_5, 'hello', [],
KeyConditions={
'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
# Test that when a table has a GSI, if the indexed attribute is missing, the
# item is added to the base table but not the index.
@pytest.mark.parametrize('op', ['PutItem', 'UpdateItem', 'BatchWriteItem'])
def test_gsi_missing_attribute(test_table_gsi_2, op):
p1 = random_string()
x1 = random_string()
p2 = random_string()
if op == 'PutItem':
test_table_gsi_2.put_item(Item={'p': p1, 'x': x1})
test_table_gsi_2.put_item(Item={'p': p2})
elif op == 'UpdateItem':
test_table_gsi_2.update_item(Key={'p': p1}, AttributeUpdates={'x': {'Value': x1, 'Action': 'PUT'}})
test_table_gsi_2.update_item(Key={'p': p2}, AttributeUpdates={})
elif op == 'BatchWriteItem':
with test_table_gsi_2.batch_writer() as batch:
batch.put_item(Item={'p': p1, 'x': x1})
batch.put_item(Item={'p': p2})
# Both items are now in the base table:
assert test_table_gsi_2.get_item(Key={'p': p1}, ConsistentRead=True)['Item'] == {'p': p1, 'x': x1}
assert test_table_gsi_2.get_item(Key={'p': p2}, ConsistentRead=True)['Item'] == {'p': p2}
# But only the first item is in the index: It can be found using a
# Query, and a scan of the index won't find it (but a scan on the base
# will).
assert_index_query(test_table_gsi_2, 'hello', [{'p': p1, 'x': x1}],
KeyConditions={'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
assert any([i['p'] == p1 for i in full_scan(test_table_gsi_2)])
# Note: with eventually consistent read, we can't really be sure that
# and item will "never" appear in the index. We do this test last,
# so if we had a bug and such item did appear, hopefully we had enough
# time for the bug to become visible. At least sometimes.
assert not any([i['p'] == p2 for i in full_scan(test_table_gsi_2, ConsistentRead=False, IndexName='hello')])
# Test when a table has a GSI, if the indexed attribute has the wrong type,
# the update operation is rejected, and is added to neither base table nor
# index. This is different from the case of a *missing* attribute, where
# the item is added to the base table but not index.
# The following three tests test_gsi_wrong_type_attribute_{put,update,batch}
# test updates using PutItem, UpdateItem, and BatchWriteItem respectively.
def test_gsi_wrong_type_attribute_put(test_table_gsi_2):
# PutItem with wrong type for 'x' is rejected, item isn't created even
# in the base table.
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_2.put_item(Item={'p': p, 'x': 3})
assert not 'Item' in test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)
def test_gsi_wrong_type_attribute_update(test_table_gsi_2):
# An UpdateItem with wrong type for 'x' is also rejected, but naturally
# if the item already existed, it remains as it was.
p = random_string()
x = random_string()
test_table_gsi_2.put_item(Item={'p': p, 'x': x})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_2.update_item(Key={'p': p}, AttributeUpdates={'x': {'Value': 3, 'Action': 'PUT'}})
assert test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'x': x}
def test_gsi_wrong_type_attribute_batchwrite(test_table_gsi_2):
# BatchWriteItem with wrong type for 'x' is rejected, item isn't created
# even in the base table.
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
with test_table_gsi_2.batch_writer() as batch:
batch.put_item({'p': p, 'x': 3})
assert not 'Item' in test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)
# Since a GSI key x cannot be a map or an array, in particular updates to
# nested attributes like x.y or x[1] are not legal. The error that DynamoDB
# reports is "Key attributes must be scalars; list random access '[]' and map
# lookup '.' are not allowed: IndexKey: x".
def test_gsi_wrong_type_attribute_update_nested(test_table_gsi_2):
p = random_string()
x = random_string()
test_table_gsi_2.put_item(Item={'p': p, 'x': x})
# We can't write a map into a GSI key column, which in this case can only
# be a string and in any case can never be a map. DynamoDB and Alternator
# both report a "type mismatch" error, exactly like in the test
# test_gsi_wrong_type_attribute_update.
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_2.update_item(Key={'p': p}, UpdateExpression='SET x = :val1',
ExpressionAttributeValues={':val1': {'a': 3, 'b': 4}})
# Here we try to set x.y for the GSI key column x. Here DynamoDB and
# Alternator produce different error messages - but both make sense.
# DynamoDB says "Key attributes must be scalars; list random access '[]'
# and map # lookup '.' are not allowed: IndexKey: x", while Alternator
# complains that "document paths not valid for this item: x.y".
with pytest.raises(ClientError, match='ValidationException'):
test_table_gsi_2.update_item(Key={'p': p}, UpdateExpression='SET x.y = :val1',
ExpressionAttributeValues={':val1': 3})
def test_gsi_wrong_type_attribute_batch(test_table_gsi_2):
# In a BatchWriteItem, if any update is forbidden, the entire batch is
# rejected, and none of the updates happen at all.
p1 = random_string()
p2 = random_string()
p3 = random_string()
items = [{'p': p1, 'x': random_string()},
{'p': p2, 'x': 3},
{'p': p3, 'x': random_string()}]
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
with test_table_gsi_2.batch_writer() as batch:
for item in items:
batch.put_item(item)
for p in [p1, p2, p3]:
assert not 'Item' in test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)
# Test when a table has a GSI, if the indexed attribute is a partition key
# in the GSI and its value is 2048 bytes, the update operation is rejected,
# and is added to neither base table nor index. DynamoDB limits partition
# keys to that length (see test_limits.py::test_limit_partition_key_len_2048)
# so wants to limit the GSI keys as well.
# Note that in test_gsi_updatetable.py we have a similar test for when adding
# a pre-existing table. In that case we can't reject the base-table update
# because the oversized attribute is already there - but can just drop this
# item from the GSI.
@pytest.mark.xfail(reason="issue #10347: key length limits not enforced")
def test_gsi_limit_partition_key_len_2048(test_table_gsi_2):
# A value for 'x' (the GSI's partition key) of length 2048 is fine:
p = random_string()
x = 'a'*2048
test_table_gsi_2.put_item(Item={'p': p, 'x': x})
assert_index_query(test_table_gsi_2, 'hello', [{'p': p, 'x': x}],
KeyConditions={
'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# PutItem with oversized for 'x' is rejected, item isn't created even
# in the base table.
p = random_string()
x = 'a'*2049
with pytest.raises(ClientError, match='ValidationException.*2048'):
test_table_gsi_2.put_item(Item={'p': p, 'x': x})
assert not 'Item' in test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)
# This is a variant of the above test, where we don't insist that the
# partition key length limit must be exactly 2048 bytes as in DynamoDB,
# but that it be *at least* 2408. I.e., we verify that 2048-byte values
# are allowed for GSI partition keys, while very long keys that surpass
# Scylla's low-level key-length limit (64 KB) are forbidden with an
# appropriate error message and not an "internal server error". This test
# should pass even if Alternator decides to adopt a different key length
# limits from DynamoDB. We do have to adopt *some* limit because the
# internal Scylla implementation has a 64 KB limit on key lengths.
@pytest.mark.xfail(reason="issue #10347: key length limits not enforced")
def test_gsi_limit_partition_key_len(test_table_gsi_2):
# A value for 'x' (the GSI's partition key) of length 2048 is fine:
p = random_string()
x = 'a'*2048
test_table_gsi_2.put_item(Item={'p': p, 'x': x})
assert_index_query(test_table_gsi_2, 'hello', [{'p': p, 'x': x}],
KeyConditions={
'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# Attribute, that is a GSI partition key, of length 64 KB + 1 is forbidden:
# it obviously exceeds DynamoDB's limit (2048 bytes), but also exceeds
# Scylla's internal limit on key length (64 KB - 1). We except to get a
# reasonable error on request validation - not some "internal server error".
# We actually used to get this "internal server error" for 64 KB - 2
# (this is probably related to issue #16772).
p = random_string()
x = 'a'*65536
with pytest.raises(ClientError, match='ValidationException.*limit'):
test_table_gsi_2.put_item(Item={'p': p, 'x': x})
assert not 'Item' in test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)
# Test when a table has a GSI, if the indexed attribute is a partition key
# in the GSI and its value is 1024 bytes, the update operation is rejected,
# and is added to neither base table nor index. DynamoDB limits partition
# keys to that length (see test_limits.py::test_limit_partition_key_len_1024)
# so wants to limit the GSI keys as well.
# Note that in test_gsi_updatetable.py we have a similar test for when adding
# a pre-existing table. In that case we can't reject the base-table update
# because the oversized attribute is already there - but can just drop this
# item from the GSI.
@pytest.mark.xfail(reason="issue #10347: key length limits not enforced")
def test_gsi_limit_sort_key_len_1024(test_table_gsi_5):
# A value for 'x' (the GSI's partition key) of length 1024 is fine:
p = random_string()
c = random_string()
x = 'a'*1024
test_table_gsi_5.put_item(Item={'p': p, 'c': c, 'x': x})
assert_index_query(test_table_gsi_5, 'hello', [{'p': p, 'c': c, 'x': x}],
KeyConditions={
'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# PutItem with oversized for 'x' is rejected, item isn't created even
# in the base table.
p = random_string()
x = 'a'*1025
with pytest.raises(ClientError, match='ValidationException.*1024'):
test_table_gsi_5.put_item(Item={'p': p, 'c': c, 'x': x})
assert not 'Item' in test_table_gsi_5.get_item(Key={'p': p, 'c': c}, ConsistentRead=True)
# This is a variant of the above test, where we don't insist that the
# partition key length limit must be exactly 1024 bytes as in DynamoDB,
# but that it be *at least* 1024. I.e., we verify that 1024-byte values
# are allowed for GSI partition keys, while very long keys that surpass
# Scylla's low-level key-length limit (64 KB) are forbidden with an
# appropriate error message and not an "internal server error". This test
# should pass even if Alternator decides to adopt a different key length
# limits from DynamoDB. We do have to adopt *some* limit because the
# internal Scylla implementation has a 64 KB limit on key lengths.
@pytest.mark.xfail(reason="issue #10347: key length limits not enforced")
def test_gsi_limit_sort_key_len(test_table_gsi_5):
# A value for 'x' (the GSI's partition key) of length 1024 is fine:
p = random_string()
c = random_string()
x = 'a'*1024
test_table_gsi_5.put_item(Item={'p': p, 'c': c, 'x': x})
assert_index_query(test_table_gsi_5, 'hello', [{'p': p, 'c': c, 'x': x}],
KeyConditions={
'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# Attribute, that is a GSI partition key, of length 64 KB + 1 is forbidden:
# it obviously exceeds DynamoDB's limit (1024 bytes), but also exceeds
# Scylla's internal limit on key length (64 KB - 1). We except to get a
# reasonable error on request validation - not some "internal server error".
# We actually used to get this "internal server error" for 64 KB - 2
# (this is probably related to issue #16772).
p = random_string()
x = 'a'*65536
with pytest.raises(ClientError, match='ValidationException.*limit'):
test_table_gsi_5.put_item(Item={'p': p, 'c': c, 'x': x})
assert not 'Item' in test_table_gsi_5.get_item(Key={'p': p, 'c': c}, ConsistentRead=True)
# A third scenario of GSI. Index has a hash key and a sort key, both are
# non-key attributes from the base table. This scenario may be very
# difficult to implement in Alternator because Scylla's materialized-views
# implementation only allows one new key column in the view, and here
# we need two (which, also, aren't actual columns, but map items).
@pytest.fixture(scope="module")
def test_table_gsi_3(dynamodb):
table = create_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'a', 'AttributeType': 'S' },
{ 'AttributeName': 'b', 'AttributeType': 'S' }
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'a', 'KeyType': 'HASH' },
{ 'AttributeName': 'b', 'KeyType': 'RANGE' }
],
'Projection': { 'ProjectionType': 'ALL' }
}
])
yield table
table.delete()
def test_gsi_3(test_table_gsi_3):
items = [{'p': random_string(), 'a': random_string(), 'b': random_string()} for i in range(10)]
with test_table_gsi_3.batch_writer() as batch:
for item in items:
batch.put_item(item)
assert_index_query(test_table_gsi_3, 'hello', [items[3]],
KeyConditions={'a': {'AttributeValueList': [items[3]['a']], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [items[3]['b']], 'ComparisonOperator': 'EQ'}})
def test_gsi_update_second_regular_base_column(test_table_gsi_3):
items = [{'p': random_string(), 'a': random_string(), 'b': random_string(), 'd': random_string()} for i in range(10)]
with test_table_gsi_3.batch_writer() as batch:
for item in items:
batch.put_item(item)
items[3]['b'] = 'updated'
test_table_gsi_3.update_item(Key={'p': items[3]['p']}, AttributeUpdates={'b': {'Value': 'updated', 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [items[3]],
KeyConditions={'a': {'AttributeValueList': [items[3]['a']], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [items[3]['b']], 'ComparisonOperator': 'EQ'}})
# Test reproducing issue #11801: In issue #5006 we noticed that in the special
# case of a GSI with with two non-key attributes as keys (test_table_gsi_3),
# an update of the second attribute forgot to delete the old row. We fixed
# that bug, but a bug remained for updates which update the value to the *same*
# value - in that case the old row shouldn't be deleted, but we did - as
# noticed in issue #11801.
def test_11801(test_table_gsi_3):
p = random_string()
a = random_string()
b = random_string()
item = {'p': p, 'a': a, 'b': b, 'd': random_string()}
test_table_gsi_3.put_item(Item=item)
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Update the attribute 'b' to the same value b that it already had.
# This shouldn't change anything in the base table or in the GSI
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'b': {'Value': b, 'Action': 'PUT'}})
assert item == test_table_gsi_3.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# In issue #11801, the following assertion failed (the view row was
# deleted and nothing matched the query).
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Above we checked that setting 'b' to the same value didn't remove
# the old GSI row. But the same update may actually modify the GSI row
# (e.g., an unrelated attribute d) - check this modification took place:
item['d'] = random_string()
test_table_gsi_3.update_item(Key={'p': p},
AttributeUpdates={'b': {'Value': b, 'Action': 'PUT'},
'd': {'Value': item['d'], 'Action': 'PUT'}})
assert item == test_table_gsi_3.get_item(Key={'p': p}, ConsistentRead=True)['Item']
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# This test is the same as test_11801, but updating the first attribute (a)
# instead of the second (b). This test didn't fail, showing that issue #11801
# is - like #5006 - specific to the case of updating the second attribute.
def test_11801_variant1(test_table_gsi_3):
p = random_string()
a = random_string()
b = random_string()
d = random_string()
item = {'p': p, 'a': a, 'b': b, 'd': d}
test_table_gsi_3.put_item(Item=item)
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': a, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# This test is the same as test_11801, but updates b to a different value
# (newb) instead of to the same one. This test didn't fail, showing that
# issue #11801 is specific to updates to the same value. This test basically
# reproduces the already-fixed #5006 (we also have another test above which
# reproduces that issue - test_gsi_update_second_regular_base_column())
def test_11801_variant2(test_table_gsi_3):
p = random_string()
a = random_string()
b = random_string()
item = {'p': p, 'a': a, 'b': b, 'd': random_string()}
test_table_gsi_3.put_item(Item=item)
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
newb = random_string()
item['b'] = newb
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'b': {'Value': newb, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [newb], 'ComparisonOperator': 'EQ'}})
# This test is the same as test_11801, but uses a different table schema
# (test_table_gsi_5) where there is only one new key column in the view (x).
# This test passed, showing that issue #11801 was specific to the special
# case of a view with two new key columns (test_table_gsi_3).
def test_11801_variant3(test_table_gsi_5):
p = random_string()
c = random_string()
x = random_string()
item = {'p': p, 'c': c, 'x': x, 'd': random_string()}
test_table_gsi_5.put_item(Item=item)
assert_index_query(test_table_gsi_5, 'hello', [item],
KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
test_table_gsi_5.update_item(Key={'p': p, 'c': c}, AttributeUpdates={'x': {'Value': x, 'Action': 'PUT'}})
assert item == test_table_gsi_5.get_item(Key={'p': p, 'c': c}, ConsistentRead=True)['Item']
assert_index_query(test_table_gsi_5, 'hello', [item],
KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# Another test similar to test_11801, but instead of updating a view key
# column to the same value it already has, simply don't update it at all
# (and just modify some other regular column). This test passed, showing
# that issue #11801 is specific to the case of updating a view key column
# to the same value it already had.
def test_11801_variant4(test_table_gsi_3):
p = random_string()
a = random_string()
b = random_string()
item = {'p': p, 'a': a, 'b': b, 'd': random_string()}
test_table_gsi_3.put_item(Item=item)
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# An update that doesn't change the GSI keys (a or b), just a regular
# column d.
item['d'] = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'d': {'Value': item['d'], 'Action': 'PUT'}})
assert item == test_table_gsi_3.get_item(Key={'p': p}, ConsistentRead=True)['Item']
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# An additional test for the case that two non-key attributes in the base
# table become keys of the view. In this test we try to cover all the
# cases of an update modifying, not modifying, or deleting, one of the two
# attributes, and checking which view rows are changed/deleted/inserted.
def test_gsi_3_long(test_table_gsi_3):
p = random_string()
a = random_string()
b = random_string()
z = random_string()
test_table_gsi_3.put_item(Item={'p': p, 'a': a, 'b': b, 'z': z})
assert_index_query(test_table_gsi_3, 'hello', [{'p': p, 'a': a, 'b': b, 'z': z}],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Change z, the existing view row changes:
z2 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'z': {'Value': z2, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [{'p': p, 'a': a, 'b': b, 'z': z2}],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Change a, the original row (a,b) is deleted, a new one (a2, b) created
a2 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': a2, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_3, 'hello', [{'p': p, 'a': a2, 'b': b, 'z': z2}],
KeyConditions={'a': {'AttributeValueList': [a2], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Change b, the original row (a2, b) is deleted, a new one (a2, b2) created
b2 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'b': {'Value': b2, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a2], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_3, 'hello', [{'p': p, 'a': a2, 'b': b2, 'z': z2}],
KeyConditions={'a': {'AttributeValueList': [a2], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b2], 'ComparisonOperator': 'EQ'}})
# Change both a and b, the old row (a2, b2) is deleted, a new one
# (a3, b3) is created
a3 = random_string()
b3 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': a3, 'Action': 'PUT'}, 'b': {'Value': b3, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a2], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b2], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_3, 'hello', [{'p': p, 'a': a3, 'b': b3, 'z': z2}],
KeyConditions={'a': {'AttributeValueList': [a3], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b3], 'ComparisonOperator': 'EQ'}})
# Delete attribute a and at the same time change b to b4. This is *not* the
# same as just not setting a (as we checked above). In this case, the
# old row should be deleted, but no new view row is created.
b4 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Action': 'DELETE'}, 'b': {'Value': b4, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a3], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b3], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a3], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b4], 'ComparisonOperator': 'EQ'}})
# Set "a" again (to a4), and the view row (a4, b4) appears
a4 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': a4, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [{'p': p, 'a': a4, 'b': b4, 'z': z2}],
KeyConditions={'a': {'AttributeValueList': [a4], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b4], 'ComparisonOperator': 'EQ'}})
# Similar to above, but for second column: Delete attribute b and at the
# same time change a to a5. the old row should be deleted, but no new
# view row is created.
a5 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'b': {'Action': 'DELETE'}, 'a': {'Value': a5, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a4], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b4], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a5], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b4], 'ComparisonOperator': 'EQ'}})
# Set "b" again (to b5), and the view row (a5, b5) appears
b5 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'b': {'Value': b5, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [{'p': p, 'a': a5, 'b': b5, 'z': z2}],
KeyConditions={'a': {'AttributeValueList': [a5], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b5], 'ComparisonOperator': 'EQ'}})
# Now unset both a and b. The view row disappears, and setting only a,
# or only b, doesn't bring it back.
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Action': 'DELETE'}, 'b': {'Action': 'DELETE'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a5], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b5], 'ComparisonOperator': 'EQ'}})
a6 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': a6, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a6], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b5], 'ComparisonOperator': 'EQ'}})
b6 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'b': {'Value': b6, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [{'p': p, 'a': a6, 'b': b6, 'z': z2}],
KeyConditions={'a': {'AttributeValueList': [a6], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b6], 'ComparisonOperator': 'EQ'}})
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Action': 'DELETE'}, 'b': {'Action': 'DELETE'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a6], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b6], 'ComparisonOperator': 'EQ'}})
b7 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'b': {'Value': b7, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a6], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b7], 'ComparisonOperator': 'EQ'}})
a7 = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'a': {'Value': a7, 'Action': 'PUT'}})
assert_index_query(test_table_gsi_3, 'hello', [{'p': p, 'a': a7, 'b': b7, 'z': z2}],
KeyConditions={'a': {'AttributeValueList': [a7], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b7], 'ComparisonOperator': 'EQ'}})
# Test that when a table has a GSI, if the indexed attribute is missing, the
# item is added to the base table but not the index.
# This is the same feature we already tested in test_gsi_missing_attribute()
# above, but on a different table: In that test we used test_table_gsi_2,
# with one indexed attribute, and in this test we use test_table_gsi_3 which
# has two base regular attributes in the view key, and more possibilities
# of which value might be missing. Reproduces issue #6008.
def test_gsi_missing_attribute_3(test_table_gsi_3):
p = random_string()
a = random_string()
b = random_string()
# First, add an item with a missing "a" value. It should appear in the
# base table, but not in the index:
test_table_gsi_3.put_item(Item={'p': p, 'b': b})
assert test_table_gsi_3.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': b}
# Note: with eventually consistent read, we can't really be sure that
# an item will "never" appear in the index. We hope that if a bug exists
# and such an item did appear, sometimes the delay here will be enough
# for the unexpected item to become visible.
assert not any([i['p'] == p for i in full_scan(test_table_gsi_3, ConsistentRead=False, IndexName='hello')])
# Same thing for an item with a missing "b" value:
test_table_gsi_3.put_item(Item={'p': p, 'a': a})
assert test_table_gsi_3.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': a}
assert not any([i['p'] == p for i in full_scan(test_table_gsi_3, ConsistentRead=False, IndexName='hello')])
# And for an item missing both:
test_table_gsi_3.put_item(Item={'p': p})
assert test_table_gsi_3.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p}
assert not any([i['p'] == p for i in full_scan(test_table_gsi_3, ConsistentRead=False, IndexName='hello')])
# A fourth scenario of GSI. Two GSIs on a single base table.
@pytest.fixture(scope="module")
def test_table_gsi_4(dynamodb):
table = create_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'a', 'AttributeType': 'S' },
{ 'AttributeName': 'b', 'AttributeType': 'S' }
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello_a',
'KeySchema': [
{ 'AttributeName': 'a', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'hello_b',
'KeySchema': [
{ 'AttributeName': 'b', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'ALL' }
}
])
yield table
table.delete()
# Test that a base table with two GSIs updates both as expected.
def test_gsi_4(test_table_gsi_4):
items = [{'p': random_string(), 'a': random_string(), 'b': random_string()} for i in range(10)]
with test_table_gsi_4.batch_writer() as batch:
for item in items:
batch.put_item(item)
assert_index_query(test_table_gsi_4, 'hello_a', [items[3]],
KeyConditions={'a': {'AttributeValueList': [items[3]['a']], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_4, 'hello_b', [items[3]],
KeyConditions={'b': {'AttributeValueList': [items[3]['b']], 'ComparisonOperator': 'EQ'}})
# Verify that describe_table lists the two GSIs.
def test_gsi_4_describe(test_table_gsi_4):
desc = test_table_gsi_4.meta.client.describe_table(TableName=test_table_gsi_4.name)
assert 'Table' in desc
assert 'GlobalSecondaryIndexes' in desc['Table']
gsis = desc['Table']['GlobalSecondaryIndexes']
assert len(gsis) == 2
assert multiset([g['IndexName'] for g in gsis]) == multiset(['hello_a', 'hello_b'])
# A scenario for GSI in which the table has both hash and sort key
@pytest.fixture(scope="module")
def test_table_gsi_5(dynamodb):
table = create_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }, { 'AttributeName': 'c', 'KeyType': 'RANGE' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'c', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'x', 'KeyType': 'RANGE' },
],
'Projection': { 'ProjectionType': 'ALL' }
}
])
yield table
table.delete()
def test_gsi_5(test_table_gsi_5):
items1 = [{'p': random_string(), 'c': random_string(), 'x': random_string()} for i in range(10)]
p1, x1 = items1[0]['p'], items1[0]['x']
p2, x2 = random_string(), random_string()
items2 = [{'p': p2, 'c': random_string(), 'x': x2} for i in range(10)]
items = items1 + items2
with test_table_gsi_5.batch_writer() as batch:
for item in items:
batch.put_item(item)
expected_items = [i for i in items if i['p'] == p1 and i['x'] == x1]
assert_index_query(test_table_gsi_5, 'hello', expected_items,
KeyConditions={'p': {'AttributeValueList': [p1], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x1], 'ComparisonOperator': 'EQ'}})
expected_items = [i for i in items if i['p'] == p2 and i['x'] == x2]
assert_index_query(test_table_gsi_5, 'hello', expected_items,
KeyConditions={'p': {'AttributeValueList': [p2], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x2], 'ComparisonOperator': 'EQ'}})
# Verify that DescribeTable correctly returns the schema of both base-table
# and secondary indexes. KeySchema is given for each of the base table and
# indexes, and AttributeDefinitions is merged for all of them together.
def test_gsi_5_describe_table_schema(test_table_gsi_5):
got = test_table_gsi_5.meta.client.describe_table(TableName=test_table_gsi_5.name)['Table']
# Copied from test_table_gsi_5 fixture
expected_base_keyschema = [
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'c', 'KeyType': 'RANGE' } ]
expected_gsi_keyschema = [
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'x', 'KeyType': 'RANGE' } ]
expected_all_attribute_definitions = [
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'c', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' } ]
assert got['KeySchema'] == expected_base_keyschema
gsis = got['GlobalSecondaryIndexes']
assert len(gsis) == 1
assert gsis[0]['KeySchema'] == expected_gsi_keyschema
# The list of attribute definitions may be arbitrarily reordered
assert multiset(got['AttributeDefinitions']) == multiset(expected_all_attribute_definitions)
# Similar DescribeTable schema test for test_table_gsi_2. The peculiarity
# in that table is that the base table has only a hash key p, and index
# only has hash key x; Now, while internally Scylla needs to add "p" as a
# clustering key in the materialized view (in Scylla the view key always
# contains the base key), when describing the table, "p" shouldn't be
# returned as a range key, because the user didn't ask for it.
# This test reproduces issue #5320.
def test_gsi_2_describe_table_schema(test_table_gsi_2):
got = test_table_gsi_2.meta.client.describe_table(TableName=test_table_gsi_2.name)['Table']
# Copied from test_table_gsi_2 fixture
expected_base_keyschema = [ { 'AttributeName': 'p', 'KeyType': 'HASH' } ]
expected_gsi_keyschema = [ { 'AttributeName': 'x', 'KeyType': 'HASH' } ]
expected_all_attribute_definitions = [
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' } ]
assert got['KeySchema'] == expected_base_keyschema
gsis = got['GlobalSecondaryIndexes']
assert len(gsis) == 1
assert gsis[0]['KeySchema'] == expected_gsi_keyschema
# The list of attribute definitions may be arbitrarily reordered
assert multiset(got['AttributeDefinitions']) == multiset(expected_all_attribute_definitions)
# This test is a comprehensive regression test for issue #5320, testing that
# DescribeTable shows the correct user-requested GSI key even when Alternator
# had to add to the underlying materialized view an "extra" clustering key
# (because Scylla's MV requires each base key column to also be a key column
# in the view). See also its LSI version in test_lsi.py.
# In test_gsi_2_describe_table_schema above we made an educated guess which
# combination of base-table and GSI keys might cause DescribeTable to return
# wrong results. In contrast, this tests rigorously checks *all* the possible
# combinations of what the base key and GSI key might be:
# * The base table's key can have one or two components (just a hash key, or
# a hash key and a range key).
# * The GSI key can also have one or two components, and each of those can
# be picked from one of the base's key columns or from a regular column.
# The test covers a grand total of 15 different cases, creating just two tables
# (for one or two base key components) - one has 5 GSIs and the second 10 GSIs.
def test_gsi_describe_table_schema_all(dynamodb):
# We have two options for the base table: it can have either have just a
# hash key (['a']) or both a hash key and a range key (['a', 'b'])
for base_keys in [ ['a'], ['a', 'b'] ]:
# The GSI key can have either one component (just hash) or two (hash
# and range). We build in gsi_keys_options a list of all the options
# for the GSI key - it's a list of vectors, each of length one or two.
gsi_keys_options=[]
# First add to gsi_keys_options all options for GSI keys with just one
# key component. It can be one of the base_keys, or some unrelated
# regular column (which we'll take as 'x')
for bk in base_keys:
gsi_keys_options.append([bk])
gsi_keys_options.append(['x'])
# Now add to gsi_keys_options GSI keys with two key component. We need
# all the ordered pairs of two different items taken from base_keys
# or two other regular columns x and y.
for pair in itertools.permutations(base_keys + ['x', 'y'], 2):
# If the key has just y, not x, it's a redundant option and we
# can drop it - the same key with just x represents the same thing.
if 'y' in pair and 'x' not in pair:
continue
# Similarly, the pair y,x is redundant - it's the same as x,y
# (note that when the base key columns are involved, the order
# does matter! a,x is not the same as x,a and we should try both)
if pair == ('y', 'x'):
continue
gsi_keys_options.append(pair)
print(f'{len(gsi_keys_options)} options for {base_keys}: {gsi_keys_options}')
# Finally, create a base table with base_keys and a bunch of GSIs with
# all the different GSI key options we collected in gsi_keys_options
if len(base_keys) == 1:
key_schema=[ { 'AttributeName': base_keys[0], 'KeyType': 'HASH' } ]
else:
key_schema=[ { 'AttributeName': base_keys[0], 'KeyType': 'HASH' },
{ 'AttributeName': base_keys[1], 'KeyType': 'RANGE' } ]
attribute_definitions = [ {'AttributeName': attr, 'AttributeType': 'S' } for attr in (base_keys + ['x', 'y']) ]
gsis = []
for i, gsi_keys in enumerate(gsi_keys_options):
if len(gsi_keys) == 1:
gsi_key_schema=[ { 'AttributeName': gsi_keys[0], 'KeyType': 'HASH' } ]
else:
gsi_key_schema=[ { 'AttributeName': gsi_keys[0], 'KeyType': 'HASH' },
{ 'AttributeName': gsi_keys[1], 'KeyType': 'RANGE' } ]
gsis.append({ 'IndexName': f'index{i}',
'KeySchema': gsi_key_schema,
'Projection': { 'ProjectionType': 'ALL' } })
with new_test_table(dynamodb,
KeySchema=key_schema,
AttributeDefinitions=attribute_definitions,
GlobalSecondaryIndexes=gsis) as table:
# Check that DescribeTable shows the table and all its GSIs correctly
got = table.meta.client.describe_table(TableName=table.name)['Table']
assert got['KeySchema'] == key_schema
got_gsis = got['GlobalSecondaryIndexes']
# We want to compare got_gsis to the original gsis, but got_gsis
# may have extra attributes that DescribeTable added beyond what
# was present in the origin table creation. So let's leave in
# got_gsis only the columns that were present in gsis[0].
got_gsis = [ {k: v for k, v in got_gsi.items() if k in gsis[0]} for got_gsi in got_gsis ]
# Use multiset to compare ignoring order
assert multiset(got_gsis) == multiset(gsis)
# All tests above involved "ProjectionType: ALL". This test checks how
# "ProjectionType:: KEYS_ONLY" works. We note that it projects both
# the index's key, *and* the base table's key. So items which had different
# base-table keys cannot suddenly become the same item in the index.
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_projection_keys_only(dynamodb):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'x', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'KEYS_ONLY' }
}
]) as table:
items = [{'p': random_string(), 'x': random_string(), 'y': random_string()} for i in range(10)]
with table.batch_writer() as batch:
for item in items:
batch.put_item(item)
wanted = ['p', 'x']
expected_items = [{k: x[k] for k in wanted if k in x} for x in items]
assert_index_scan(table, 'hello', expected_items)
# Test for "ProjectionType: INCLUDE". The secondary table includes the
# its own and the base's keys (as in KEYS_ONLY) plus the extra keys given
# in NonKeyAttributes.
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_projection_include(dynamodb):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'x', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'INCLUDE',
'NonKeyAttributes': ['a', 'b'] }
}
]) as table:
# Some items have the projected attributes a,b and some don't:
items = [{'p': random_string(), 'x': random_string(), 'a': random_string(), 'b': random_string(), 'y': random_string()} for i in range(10)]
items = items + [{'p': random_string(), 'x': random_string(), 'y': random_string()} for i in range(10)]
with table.batch_writer() as batch:
for item in items:
batch.put_item(item)
wanted = ['p', 'x', 'a', 'b']
expected_items = [{k: x[k] for k in wanted if k in x} for x in items]
assert_index_scan(table, 'hello', expected_items)
print(len(expected_items))
# Despite the name "NonKeyAttributes", key attributes *may* be listed.
# But they have no effect - because key attributes are always projected
# anyway.
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_projection_include_keyattributes(dynamodb):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' } ],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'x', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'INCLUDE',
'NonKeyAttributes': ['a', 'x'] }
}
]) as table:
p = random_string();
x = random_string();
a = random_string();
b = random_string();
table.put_item(Item={'p': p, 'x': x, 'a': a, 'b': b})
# We expect both key attributes ('p' and 'x') to be projected, as
# well as 'a' listed on NonKeyAttributes. The fact that 'x' was
# also listed on NonKeyAttributes and 'p' wasn't doesn't make a
# difference. The non-key 'b' wasn't listed, so it will not be
# retrieved from the GSI.
expected_items = [{'p': p, 'x': x, 'a': a}]
assert_index_scan(table, 'hello', expected_items)
# In this test, we add two GSIs, one projecting the other GSI's key.
# This is an interesting case in Alternator's implementation, because
# GSI keys currently become actual Scylla columns - while regular attributes
# do not (they are elements of a single map column), so we need to remember
# to project both real columns and map elements into the view.
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_projection_include_otherkey(dynamodb):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
{ 'AttributeName': 'a', 'AttributeType': 'S' },
{ 'AttributeName': 'z', 'AttributeType': 'S' } ],
GlobalSecondaryIndexes=[
{ 'IndexName': 'indexx',
'KeySchema': [
{ 'AttributeName': 'x', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'INCLUDE',
'NonKeyAttributes': ['a', 'b'] }
},
{ 'IndexName': 'indexa',
'KeySchema': [
{ 'AttributeName': 'a', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'KEYS_ONLY' }
},
{ 'IndexName': 'indexz',
'KeySchema': [
{ 'AttributeName': 'z', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'KEYS_ONLY' }
}
]) as table:
p = random_string();
x = random_string();
a = random_string();
b = random_string();
c = random_string();
z = random_string();
table.put_item(Item={'p': p, 'x': x, 'a': a, 'b': b, 'c': c, 'z': z})
# When scanning indexx, we expect both its key attributes ('x')
# and the base table's ('p') to be projected, as well as 'a' and 'b'
# listed on NonKeyAttributes. 'c' isn't projected, and neither is 'z'
# despite being some other GSI's key. Note that the projected 'a'
# also happens to be some other GSI's key, while 'b' isn't, allowing
# us to exercise both code paths (Alternator stores regular attributes
# differently from attributes which are keys of some GSI).
expected_items = [{'p': p, 'x': x, 'a': a, 'b': b}]
assert_index_scan(table, 'indexx', expected_items)
# With ProjectionType=INCLUDE, NonKeyAttributes must not be missing:
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_projection_error_missing_nonkeyattributes(dynamodb):
with pytest.raises(ClientError, match='ValidationException.*NonKeyAttributes'):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'x', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'INCLUDE' }
}
]) as table:
pass
# With ProjectionType!=INCLUDE, NonKeyAttributes must not be present:
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_projection_error_superflous_nonkeyattributes(dynamodb):
with pytest.raises(ClientError, match='ValidationException.*NonKeyAttributes'):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'x', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'ALL',
'NonKeyAttributes': ['a'] }
}
]) as table:
pass
# Duplicate attribute names in NonKeyAttributes of INCLUDE are not allowed:
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_projection_error_duplicate(dynamodb):
with pytest.raises(ClientError, match='ValidationException.*Duplicate'):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'x', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'INCLUDE',
'NonKeyAttributes': ['a', 'a'] }
}
]) as table:
pass
# NonKeyAttributes must be a list of strings. Non-strings in this list
# result, for some reason, in SerializationException instead of the more
# usual ValidationException.
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_projection_error_nonstring_nonkeyattributes(dynamodb):
with pytest.raises(ClientError, match='SerializationException'):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [
{ 'AttributeName': 'x', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'INCLUDE',
'NonKeyAttributes': ['a', 123] }
}
]) as table:
pass
# An unsupported ProjectionType value should result in an error:
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_bad_projection_type(dynamodb):
with pytest.raises(ClientError, match='ValidationException.*nonsense'):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }],
AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [{ 'AttributeName': 'p', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'nonsense' }
}
]) as table:
pass
# DynamoDB's says the "Projection" argument of GlobalSecondaryIndexes is
# mandatory, and indeed Boto3 enforces that it must be passed. The
# documentation then goes on to claim that the "ProjectionType" member of
# "Projection" is optional - and Boto3 allows it to be missing. But in
# fact, it is not allowed to be missing: DynamoDB complains: "Unknown
# ProjectionType: null".
@pytest.mark.xfail(reason="GSI projection not supported - issue #5036")
def test_gsi_missing_projection_type(dynamodb):
with pytest.raises(ClientError, match='ValidationException.*ProjectionType'):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }],
AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [{ 'AttributeName': 'p', 'KeyType': 'HASH' }],
'Projection': {}
}
]) as table:
pass
# Utility function for creating a new table a GSI with the given name,
# and, if creation was successful, delete it. Useful for testing which
# GSI names work.
def create_gsi(dynamodb, index_name):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }],
AttributeDefinitions=[{ 'AttributeName': 'p', 'AttributeType': 'S' }],
GlobalSecondaryIndexes=[
{ 'IndexName': index_name,
'KeySchema': [{ 'AttributeName': 'p', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
}
]) as table:
# Verify that the GSI wasn't just ignored, as Scylla originally did ;-)
assert 'GlobalSecondaryIndexes' in table.meta.client.describe_table(TableName=table.name)['Table']
# Like table names (tested in test_table.py), index names must must also
# be 3-255 characters and match the regex [a-zA-Z0-9._-]+. This test
# is similar to test_create_table_unsupported_names(), but for GSI names.
# Note that Scylla is actually more limited in the length of the index
# names, because both table name and index name, together, have to fit in
# 221 characters. But we don't verify here this specific limitation.
def test_gsi_unsupported_names(dynamodb):
with pytest.raises(ClientError, match='ValidationException.*3'):
create_gsi(dynamodb, 'n')
with pytest.raises(ClientError, match='ValidationException.*3'):
create_gsi(dynamodb, 'nn')
with pytest.raises(ClientError, match='ValidationException.*nnnnn'):
create_gsi(dynamodb, 'n' * 256)
with pytest.raises(ClientError, match='ValidationException.*nyh'):
create_gsi(dynamodb, 'nyh@test')
# On the other hand, names following the above rules should be accepted. Even
# names which the Scylla rules forbid, such as a name starting with .
def test_gsi_non_scylla_name(dynamodb):
create_gsi(dynamodb, '.alternator_test')
# Index names with 255 characters are allowed in Dynamo. In Scylla, the
# limit is different - the sum of both table and index length plus an extra 1
# cannot exceed 222 characters.
# (compare test_create_and_delete_table_255/222()).
@pytest.mark.xfail(reason="Alternator limits table name length + GSI name length to 221")
def test_gsi_very_long_name_255(dynamodb):
create_gsi(dynamodb, 'n' * 255)
def test_gsi_very_long_name_256(dynamodb):
with pytest.raises(ClientError, match='ValidationException'):
create_gsi(dynamodb, 'n' * 256)
def test_gsi_very_long_name_222(dynamodb, scylla_only):
# If we subtract from 222 the table's name length (we assume that
# unique_table_name() always returns the same length) and an extra 1,
# this is how long the GSI's name may be:
max = 222 - len(unique_table_name()) - 1
# This max length should work:
create_gsi(dynamodb, 'n' * max)
# But a name one byte longer should fail:
with pytest.raises(ClientError, match='ValidationException.*total length'):
create_gsi(dynamodb, 'n' * (max+1))
# Verify that ListTables does not list materialized views used for indexes.
# This is hard to test, because we don't really know which table names
# should be listed beyond those we created, and don't want to assume that
# no other test runs in parallel with us. So the method we chose is to use a
# unique random name for an index, and check that no table contains this
# name. This assumes that materialized-view names are composed using the
# index's name (which is currently what we do).
@pytest.fixture(scope="module")
def test_table_gsi_random_name(dynamodb):
index_name = random_string()
table = create_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'c', 'KeyType': 'RANGE' }
],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'c', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': index_name,
'KeySchema': [
{ 'AttributeName': 'c', 'KeyType': 'HASH' },
{ 'AttributeName': 'p', 'KeyType': 'RANGE' },
],
'Projection': { 'ProjectionType': 'ALL' }
}
],
)
yield [table, index_name]
table.delete()
def test_gsi_list_tables(dynamodb, test_table_gsi_random_name):
table, index_name = test_table_gsi_random_name
# Check that the random "index_name" isn't a substring of any table name:
tables = list_tables(dynamodb)
for name in tables:
assert not index_name in name
# But of course, the table's name should be in the list:
assert table.name in tables
# Test the "Select" parameter of a Query on a GSI. We have in test_query.py
# a test 'test_query_select' for this parameter on a query of a normal (base)
# table, but for GSI and LSI the ALL_PROJECTED_ATTRIBUTES is additionally
# allowed, and we want to test it. Moreover, in a GSI, when only a subset of
# the attributes were projected into the GSI, it is impossible to request
# that Select return other attributes.
# We split the test into two, the first just requiring proper implementation
# of Select, and the second requiring also a proper implementation of
# projection of just a subset of the attributes.
def test_gsi_query_select_1(test_table_gsi_1):
items = [{'p': random_string(), 'c': random_string(), 'x': random_string(), 'y': random_string()} for i in range(10)]
with test_table_gsi_1.batch_writer() as batch:
for item in items:
batch.put_item(item)
c = items[0]['c']
expected_items = [x for x in items if x['c'] == c]
assert_index_query(test_table_gsi_1, 'hello', expected_items,
KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
# Unlike in base tables, here Select=ALL_PROJECTED_ATTRIBUTES is
# allowed, and in this case (all attributes are projected into this
# index) returns all attributes.
assert_index_query(test_table_gsi_1, 'hello', expected_items,
Select='ALL_PROJECTED_ATTRIBUTES',
KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
# Because in this GSI all attributes are projected into the index,
# ALL_ATTRIBUTES is allowed as well. And so is SPECIFIC_ATTRIBUTES
# (with AttributesToGet / ProjectionExpression) for any attributes,
# and of course so is COUNT.
assert_index_query(test_table_gsi_1, 'hello', expected_items,
Select='ALL_ATTRIBUTES',
KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
expected_items = [{'y': x['y']} for x in items if x['c'] == c]
assert_index_query(test_table_gsi_1, 'hello', expected_items,
Select='SPECIFIC_ATTRIBUTES',
AttributesToGet=['y'],
KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
assert not 'Items' in test_table_gsi_1.query(ConsistentRead=False,
IndexName='hello',
Select='COUNT',
KeyConditions={'c': {'AttributeValueList': [c], 'ComparisonOperator': 'EQ'}})
@pytest.mark.xfail(reason="Projection not supported yet. Issue #5036")
def test_gsi_query_select_2(dynamodb):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'hello',
'KeySchema': [ { 'AttributeName': 'x', 'KeyType': 'HASH' } ],
'Projection': { 'ProjectionType': 'INCLUDE',
'NonKeyAttributes': ['a'] }
}
]) as table:
items = [{'p': random_string(), 'x': random_string(), 'a': random_string(), 'b': random_string()} for i in range(10)]
with table.batch_writer() as batch:
for item in items:
batch.put_item(item)
x = items[0]['x']
# Unlike in base tables, here Select=ALL_PROJECTED_ATTRIBUTES is
# allowed, and only the projected attributes are returned (in this
# case the key of both base and GSI ('p' and 'x') and 'a' - but not
# 'b'. Moreover, it is the default if Select isn't specified at all.
expected_items = [{'p': z['p'], 'x': z['x'], 'a': z['a']} for z in items if z['x'] == x]
assert_index_query(table, 'hello', expected_items,
KeyConditions={'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
assert_index_query(table, 'hello', expected_items,
Select='ALL_PROJECTED_ATTRIBUTES',
KeyConditions={'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# Because in this GSI not all attributes are projected into the index,
# Select=ALL_ATTRIBUTES is *not* allowed.
with pytest.raises(ClientError, match='ValidationException.*ALL_ATTRIBUTES'):
assert_index_query(table, 'hello', expected_items,
Select='ALL_ATTRIBUTES',
KeyConditions={'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# SPECIFIC_ATTRIBUTES (with AttributesToGet / ProjectionExpression)
# is allowed for the projected attributes, but not for unprojected
# attributes.
expected_items = [{'a': z['a']} for z in items if z['x'] == x]
assert_index_query(table, 'hello', expected_items,
Select='SPECIFIC_ATTRIBUTES',
AttributesToGet=['a'],
KeyConditions={'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# Select=COUNT is also allowed, and doesn't return item content
assert not 'Items' in table.query(ConsistentRead=False,
IndexName='hello',
Select='COUNT',
KeyConditions={'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# In all GSIs above, the GSI's key was a string from the base table
# (AttributeType: S). While most of the GSI functionality is independent of
# the key's type, some may be type-specific - Alternator may rely on a
# "computed function" to deserializes the value from the base to put it in
# the view row. So test_table_gsi_6 is a table which has six GSIs, each one
# with one of the three legal AttributeType (S, B, and N) for its partition
# key or clustering key.
@pytest.fixture(scope="module")
def test_table_gsi_6(dynamodb):
table = create_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' } ],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 's', 'AttributeType': 'S' },
{ 'AttributeName': 'b', 'AttributeType': 'B' },
{ 'AttributeName': 'n', 'AttributeType': 'N' }
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'gsi_s',
'KeySchema': [
{ 'AttributeName': 's', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'gsi_ss',
'KeySchema': [
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 's', 'KeyType': 'RANGE' },
],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'gsi_b',
'KeySchema': [
{ 'AttributeName': 'b', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'gsi_sb',
'KeySchema': [
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'b', 'KeyType': 'RANGE' },
],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'gsi_n',
'KeySchema': [
{ 'AttributeName': 'n', 'KeyType': 'HASH' },
],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'gsi_sn',
'KeySchema': [
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'n', 'KeyType': 'RANGE' },
],
'Projection': { 'ProjectionType': 'ALL' }
}
])
yield table
table.delete()
# Tests for test_table_gsi_6, just checking that the write of different types
# doesn't fail (below we'll have additional tests that also read the GSI).
# Also check failure cases (wrong types, and empty string).
# As explained in a comment above for test_table_gsi_6, the main goal of these
# tests is to check the "computed function" which our implementation uses to
# parse the base-table values of different types.
def test_gsi_6_write_s(test_table_gsi_6):
test_table_gsi_6.put_item(Item={'p': random_string(), 's': random_string()})
def test_gsi_6_write_s_fail(test_table_gsi_6):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*empty'):
test_table_gsi_6.put_item(Item={'p': p, 's': ''})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 's': 3})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 's': b'hi'})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 's': True})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 's': {'hi', 'there'}})
def test_gsi_6_write_b(test_table_gsi_6):
p = random_string()
b = random_bytes()
test_table_gsi_6.put_item(Item={'p': p, 'b': b})
def test_gsi_6_write_b_fail(test_table_gsi_6):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*empty'):
test_table_gsi_6.put_item(Item={'p': p, 'b': b''})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 'b': 3})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 'b': 'hi'})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 'b': True})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 'b': {'hi', 'there'}})
def test_gsi_6_write_n(test_table_gsi_6):
p = random_string()
n = 87
test_table_gsi_6.put_item(Item={'p': p, 'n': n})
def test_gsi_6_write_n_fail(test_table_gsi_6):
p = random_string()
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 'n': b'hi'})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 'n': 'hi'})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 'n': True})
with pytest.raises(ClientError, match='ValidationException.*mismatch'):
test_table_gsi_6.put_item(Item={'p': p, 'n': {'hi', 'there'}})
def test_gsi_6(test_table_gsi_6):
p = random_string()
s = random_string()
b = random_bytes()
n = 17
x = random_string()
item={'p': p, 's': s, 'b': b, 'n': n, 'x': x}
test_table_gsi_6.put_item(Item=item)
# Check that all six GSIs as usable with their different keys.
# Note that we're reading from six different GSIs, so we need to use
# the maybe-retrying assert_index_query() for each of them.
assert_index_query(test_table_gsi_6, 'gsi_s', [item],
KeyConditions={'s': {'AttributeValueList': [s], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_6, 'gsi_ss', [item],
KeyConditions={'s': {'AttributeValueList': [s], 'ComparisonOperator': 'EQ'}, 'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_6, 'gsi_b', [item],
KeyConditions={'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_6, 'gsi_sb', [item],
KeyConditions={'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}, 'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_6, 'gsi_n', [item],
KeyConditions={'n': {'AttributeValueList': [n], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_6, 'gsi_sn', [item],
KeyConditions={'n': {'AttributeValueList': [n], 'ComparisonOperator': 'EQ'}, 'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}})
# Test that trying to create a table with two GSIs with the same name is an
# error.
# In test_gsi_backfill we also check that adding a second GSI with the same
# name later, after the already exists table exists, is also an error.
# See also test_lsi.py::test_lsi_and_gsi_same_same which shows even GSIs
# and LSIs may not have the same name (and explains why)
def test_gsi_same_name_forbidden(dynamodb):
with pytest.raises(ClientError, match='ValidationException.*Duplicate.*index1'):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
{ 'AttributeName': 'y', 'AttributeType': 'S' }
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'index1',
'KeySchema': [{ 'AttributeName': 'x', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'index1', # different index, samed name!
'KeySchema': [{ 'AttributeName': 'y', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
}
]) as table:
pass
# Even if the two indexes are identical twins - having the same definition
# exactly - it's not allowed.
with pytest.raises(ClientError, match='ValidationException.*Duplicate.*index1'):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'index1',
'KeySchema': [{ 'AttributeName': 'x', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'index1',
'KeySchema': [{ 'AttributeName': 'x', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
}
]) as table:
pass
# Trying to create two differently-named GSIs in the same table indexing
# the same attribute with exactly the same parameters is redundant, but
# allowed, and works (and not recommended because it is very wasteful...)
def test_gsi_duplicate_with_different_name(dynamodb):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'index1',
'KeySchema': [{ 'AttributeName': 'x', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'index2', # same index, different name
'KeySchema': [{ 'AttributeName': 'x', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
}
]) as table:
pass
# But, trying to create two different GSIs with different type for the same
# key is NOT allowed. The reason is that DynamoDB wants to insist that future
# writes to this attribute must have the declared type - and can't insist on
# two different types at the same time.
# We have two versions of this test: One here when the conflict happens during
# the table creation, and one in test_gsi_updatetable.py where the second GSI
# is added after the table already exists with the first GSI.
# Reproduces #13870 (see also test_create_table_duplicate_attribute_name in
# test_table.py).
def test_gsi_key_type_conflict_on_create(dynamodb):
with pytest.raises(ClientError, match='ValidationException.*uplicate.*xyz'):
with new_test_table(dynamodb,
KeySchema=[ { 'AttributeName': 'p', 'KeyType': 'HASH' }],
# Note how because how this API works, having two different types
# for the same attribute 'xyz' looks very strange (there is not even
# a way to say which definition applies to which index) so
# unsurprisingly it's not accepted.
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'xyz', 'AttributeType': 'S' },
{ 'AttributeName': 'xyz', 'AttributeType': 'N' },
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'index1',
'KeySchema': [{ 'AttributeName': 'xyz', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
},
{ 'IndexName': 'index2',
'KeySchema': [{ 'AttributeName': 'xyz', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
}
]) as table:
pass
# Test similar to test_11801 and test_11801_variant2, but this test first
# updates the range key b to a new value (like variant2) and then sets it
# back to its original value. It reproduces issue #17119 - the last
# modification was lost because the wrong timestamp was used.
# The bug is specific to the case that the GSI has two non-key columns
# as its keys, so we test it on test_table_gsi_3 which has this feature.
def test_17119(test_table_gsi_3):
p = random_string()
a = random_string()
b = random_string()
item = {'p': p, 'a': a, 'b': b, 'd': random_string()}
test_table_gsi_3.put_item(Item=item)
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Change the GSI range key b to a different value newb.
newb = random_string()
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'b': {'Value': newb, 'Action': 'PUT'}})
item['b'] = newb
assert item == test_table_gsi_3.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# The item newb should appear in the GSI, item b should be gone:
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [newb], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# Change the GSI range key b back to its original value. Item newb
# should disappear from the GSI, and item b should reappear:
test_table_gsi_3.update_item(Key={'p': p}, AttributeUpdates={'b': {'Value': b, 'Action': 'PUT'}})
item['b'] = b
assert item == test_table_gsi_3.get_item(Key={'p': p}, ConsistentRead=True)['Item']
assert_index_query(test_table_gsi_3, 'hello', [],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [newb], 'ComparisonOperator': 'EQ'}})
# This assertion failed in issue #17119:
assert_index_query(test_table_gsi_3, 'hello', [item],
KeyConditions={'a': {'AttributeValueList': [a], 'ComparisonOperator': 'EQ'},
'b': {'AttributeValueList': [b], 'ComparisonOperator': 'EQ'}})
# This test is like test_17119 above, just in a table with just one new
# key column in the GSI. The bug of #17119 doesn't reproduce here, showing
# the problem was specific to the case of two new GSI key columns.
def test_17119a(test_table_gsi_2):
p = random_string()
x = random_string()
item = {'p': p, 'x': x, 'z': random_string()}
test_table_gsi_2.put_item(Item=item)
assert_index_query(test_table_gsi_2, 'hello', [item],
KeyConditions={'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# Change the GSI range key x to a different value.
newx = random_string()
test_table_gsi_2.update_item(Key={'p': p}, AttributeUpdates={'x': {'Value': newx, 'Action': 'PUT'}})
item['x'] = newx
assert item == test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)['Item']
# The item newx should appear in the GSI, item x should be gone:
assert_index_query(test_table_gsi_2, 'hello', [item],
KeyConditions={'x': {'AttributeValueList': [newx], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_2, 'hello', [],
KeyConditions={'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# Change the GSI range key x back to its original value. Item newx
# should disappear from the GSI, and item x should reappear:
test_table_gsi_2.update_item(Key={'p': p}, AttributeUpdates={'x': {'Value': x, 'Action': 'PUT'}})
item['x'] = x
assert item == test_table_gsi_2.get_item(Key={'p': p}, ConsistentRead=True)['Item']
assert_index_query(test_table_gsi_2, 'hello', [],
KeyConditions={'x': {'AttributeValueList': [newx], 'ComparisonOperator': 'EQ'}})
assert_index_query(test_table_gsi_2, 'hello', [item],
KeyConditions={'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# The following test checks what happens when we have an LSI and a GSI using
# the same base-table attribute. We want to verify that if the implementation
# happens to store key attributes for LSIs and GSIs differently, and we have
# both on the same table, both are usable. This is a delicate corner case,
# which should be tested so it doesn't regress.
#
# In particular, we plan LSI and GSI to be implemented as follows: When "x"
# is an LSI key, "x" becomes a full-fleged column in the schema (remember
# that in DynamoDB, an LSI must be defined at table-creation time). Yet, when
# "x" is a GSI key, it will use a "computed column" to extract x from the
# map ":attrs" of un-schema'ed attributes. So do the two work at the same time?
# It turns out the code value_getter::operator() which is supposed to
# read the base data from either a real column or a computed column,
# looks for the real column *first*; If it exists (as in the case of the
# LSI), it is used and the computed column is outright ignored. Because
# this logic is delicate and even counter-intuitive and at risk of being
# "cleaned up" in the future, this test is an important regression test.
# By the way, we have in test_lsi.py::test_lsi_and_gsi() another test
# for combining GSI and LSI, testing somewhat different things.
def test_gsi_and_lsi_same_key(dynamodb):
# A table whose non-key column "x" serves as a range key in an LSI,
# and partition key in a GSI.
with new_test_table(dynamodb,
KeySchema=[
# Must have both hash key and range key to allow LSI creation
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'c', 'KeyType': 'RANGE' }
],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'c', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': 'S' },
],
LocalSecondaryIndexes=[
{ 'IndexName': 'lsi',
'KeySchema': [
{ 'AttributeName': 'p', 'KeyType': 'HASH' },
{ 'AttributeName': 'x', 'KeyType': 'RANGE' },
],
'Projection': { 'ProjectionType': 'ALL' }
}
],
GlobalSecondaryIndexes=[
{ 'IndexName': 'gsi',
'KeySchema': [{ 'AttributeName': 'x', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
}
]) as table:
# Write some data with attribute x, see it appear in the base table
# and both LSI and GSI.
p = random_string() # key column in base (and therefore in GSI & LSI)
c = random_string() # key column in base (and therefore in GSI & LSI)
x = random_string() # key column in LSI and GSI (regular in base)
y = random_string() # not a key anywhere
item = {'p': p, 'c': c, 'x': x, 'y': y}
table.put_item(Item=item)
assert table.get_item(Key={'p': p, 'c': c}, ConsistentRead=True)['Item'] == item
assert_index_query(table, 'lsi', [item],
KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}, 'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
assert_index_query(table, 'gsi', [item],
KeyConditions={'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# Check that any type besides S(tring), B(ytes) or N(umber)s is *not* NOT
# allowed as the type of a GSI key attribute. We don't check here that these
# three types *are* allowed, because we already checked this in other tests
# (see test_gsi_6_*).
def test_gsi_invalid_key_types(dynamodb):
# The following are all the types that DynamoDB supports, as documented in
# https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.LowLevelAPI.html
# also the non-existent type "junk" yields the same error message.
for key_type in ['BOOL', 'NULL', 'M', 'L', 'SS', 'NS', 'BS', 'junk']:
# DynamDB's and Alternator's error messages are different, but both
# include the invalid type's name in single quotes.
with pytest.raises(ClientError, match=f"ValidationException.*'{key_type}'"):
with new_test_table(dynamodb,
KeySchema=[{ 'AttributeName': 'p', 'KeyType': 'HASH' }],
AttributeDefinitions=[
{ 'AttributeName': 'p', 'AttributeType': 'S' },
{ 'AttributeName': 'x', 'AttributeType': key_type },
],
GlobalSecondaryIndexes=[{
'IndexName': 'gsi',
'KeySchema': [{ 'AttributeName': 'x', 'KeyType': 'HASH' }],
'Projection': { 'ProjectionType': 'ALL' }
}]) as table:
pass
# In test_table_gsi_2, the base table has just a hash key "p", and the GSI
# has just the hash key "x". The materialized view that Alternator uses to
# implement this GSI needs to add "p" as an extra clustering key, but it
# doesn't mean that "p" should be allowed in a KeyConditions or
# KeyConditionExpression - it shouldn't because it's not a real range key.
@pytest.mark.xfail(reason="Issue #26103")
def test_faux_range_key_in_keyconditions(test_table_gsi_2):
p = random_string()
x = random_string()
item = {'p': p, 'x': x, 'z': random_string()}
test_table_gsi_2.put_item(Item=item)
# The GSI 'hello' has just "x" as a hash key, so it can be used in a
# KeyConditions in Query:
assert_index_query(test_table_gsi_2, 'hello', [item],
KeyConditions={ 'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# "p" is not a range key of this GSI, so it cannot be used in a
# KeyConditions, and should be an error.
with pytest.raises(ClientError, match='ValidationException.*key condition'):
assert_index_query(test_table_gsi_2, 'hello', [item],
KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
# "z" is not a key of the GSI, so obviously the following command
# must not work, but let's also check that the error message mentions
# the write thing, not the irrelevant "p". The DynamoDB error message
# is just "Query key condition not supported".
with pytest.raises(ClientError, match='ValidationException.*key condition'):
assert_index_query(test_table_gsi_2, 'hello', [item],
KeyConditions={'z': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'},
'x': {'AttributeValueList': [x], 'ComparisonOperator': 'EQ'}})
@pytest.mark.xfail(reason="Issue #26103")
def test_faux_range_key_in_keyconditionexpression(test_table_gsi_2):
p = random_string()
x = random_string()
item = {'p': p, 'x': x, 'z': random_string()}
test_table_gsi_2.put_item(Item=item)
# The GSI 'hello' has just "x" as a hash key, so it can be used in a
# KeyConditionExpression in Query:
assert_index_query(test_table_gsi_2, 'hello', [item],
KeyConditionExpression='x=:x',
ExpressionAttributeValues={':x': x})
# "p" is not a range key of this GSI, so it cannot be used in a
# KeyConditionExpression, and should be an error.
with pytest.raises(ClientError, match='ValidationException.*key condition'):
assert_index_query(test_table_gsi_2, 'hello', [item],
KeyConditionExpression='x=:x AND p=:p',
ExpressionAttributeValues={':x': x, ':p': p})
# "z" is not a key of the GSI, so obviously the following command
# must not work, but let's also check that the error message mentions
# the write thing, not the irrelevant "p". The DynamoDB error message
# is just "Query key condition not supported".
with pytest.raises(ClientError, match='ValidationException.*key condition'):
assert_index_query(test_table_gsi_2, 'hello', [item],
KeyConditionExpression='x=:x AND z=:z',
ExpressionAttributeValues={':x': x, ':z': p})
# While in test_scan.py and test_query.py we have extensive tests that
# paging works for Scan and Query, the paging on a GSI might in theory
# be implemented differently - an in particular ExclusiveStartKey might
# be represented differently - so we should check it explicitly too.
# For simplicity, we'll test here just Query, not Scan.
def test_gsi_query_paging(test_table_gsi_5):
p = random_string()
items = [{'p': p, 'c': str(i), 'x': str(i%3)} for i in range(23)]
with test_table_gsi_5.batch_writer() as batch:
for item in items:
batch.put_item(item)
for limit in [1, 2, 17, 100]:
# GSIs are eventually consistent, so we may need to retry our scan
# but usually, we won't.
timeout = time.time() + 10
while time.time() < timeout:
query_args = {
'ConsistentRead': False,
'IndexName': 'hello',
'KeyConditions': {'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}},
'Limit': limit
}
response = test_table_gsi_5.query(**query_args)
got_items = response['Items']
assert len(response['Items']) <= limit
while 'LastEvaluatedKey' in response:
response = test_table_gsi_5.query(ExclusiveStartKey=response['LastEvaluatedKey'], **query_args)
got_items.extend(response['Items'])
assert len(response['Items']) <= limit
if multiset(items) == multiset(got_items):
# Test (for this limit) done successfully
break
time.sleep(0.1)
assert multiset(items) == multiset(got_items)
# The previous test (test_gsi_query_paging) showed that paging a Query
# on a GSI works correctly. In particular it means that each page returned an
# ExclusiveStartKey that was successfully used to read the next page.
# Adding debugging printouts of ExclusiveStartKey, we noticed that it doesn't
# just specify the key of the GSI that we are scanning - (p, x) - but actually
# has (p, x, c). In other words the scan process needs all the base table key
# columns, not just the GSI key columns, to know where it is in the scan
# process. This is no surprise for Alternator, because in our implementation
# (p, x, c) is actually the key we use for the materialized view - the
# view's key can't be just (p, x).
# Anyway, the point of the following test is to verify that we can indeed
# pass (p, x, c) as ExclusiveStartKey and it will be able to jump to that
# position - even if we're not in the middle of paging.
# This test is analogous to test_query.py::test_query_exclusivestartkey
# just for reading from a GSI instead of a base table.
#
# Note that another thing that this test demonstrates is that it is not
# really useful for a user to build ExclusiveStartKey manually when
# scanning a GSI. The problem is that the sort order of the extra tie
# column - c - is NOT defined in DynamoDB. In Alternator, it is also sorted
# like a second sort key, but this is not true in DynamoDB and the sorting
# of these ties is unspecified, and empirically - not in sorted order.
# So the user is unlikely to know how to invent the 'c' value for
# ExclusiveStartKey, which makes this feature not very useful for anything
# besides paging.
def test_gsi_query_exclusivestartkey(test_table_gsi_5):
p = random_string()
items = [{'p': p, 'c': str(i), 'x': str(i%3)} for i in range(23)]
with test_table_gsi_5.batch_writer() as batch:
for item in items:
batch.put_item(item)
# GSIs are eventually consistent, so we may need to retry our scan
# but usually, we won't.
timeout = time.time() + 10
while time.time() < timeout:
exclusive_start_key = {'p': p, 'x': '1', 'c': '7'}
query_args = {
'ConsistentRead': False,
'IndexName': 'hello',
'KeyConditions': {'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}},
}
response = test_table_gsi_5.query(
ExclusiveStartKey=exclusive_start_key, **query_args)
got_items = response['Items']
# Now we have a problem - we don't know which order to expect items
# to be returned. We do know that items are sorted by x, but don't
# know how items tied on x will be sorted! In Scylla, the tied items
# will be sorted by "c" (the second clustering key of the view),
# but it turns out that this is NOT guaranteed in DynamoDB and
# the tied items are ordered in an unknown way.
# So what we can do is the read *all* the items from the GSI (without
# an ExclusiveStartKey), and then verify that ExclusiveStartKey
# started in the right place of that all_gsi_items.
all_gsi_items = test_table_gsi_5.query(**query_args)['Items']
if multiset(all_gsi_items) != multiset(items):
# we didn't read yet all items, need to retry
time.sleep(0.1)
continue
index = all_gsi_items.index(exclusive_start_key)
expected_items = all_gsi_items[index+1:]
if multiset(expected_items) == multiset(got_items):
# Test done successfully
break
time.sleep(0.1)
assert multiset(expected_items) == multiset(got_items)
# In the previous test (test_gsi_query_exclusivestartkey) we should that
# ExclusiveStartKey contain all the base and GSI key columns (in our
# case - p,c,x). Here we verify that it is not allowed for any one of
# these columns to be missing in ExclusiveStartKey
def test_gsi_query_exclusivestartkey_missing_column(test_table_gsi_5):
# The error that DynamoDB reports if the sort key is missing in
# ExclusiveStartKey is "The provided starting key is invalid". In
# Alternator, the error is "Key column c not found".
with pytest.raises(ClientError, match='ValidationException.*[kK]ey'):
test_table_gsi_5.query(
ConsistentRead=False,
IndexName='hello',
KeyConditions={'p': { 'AttributeValueList': ['dog'], 'ComparisonOperator': 'EQ'}},
# missing 'c':
ExclusiveStartKey= { 'p': 'dog', 'x': 'cat' })
with pytest.raises(ClientError, match='ValidationException.*[kK]ey'):
test_table_gsi_5.query(
ConsistentRead=False,
IndexName='hello',
KeyConditions={'p': { 'AttributeValueList': ['dog'], 'ComparisonOperator': 'EQ'}},
# missing 'x':
ExclusiveStartKey= { 'p': 'dog', 'c': 'cat' })
with pytest.raises(ClientError, match='ValidationException.*[kK]ey'):
test_table_gsi_5.query(
ConsistentRead=False,
IndexName='hello',
KeyConditions={'p': { 'AttributeValueList': ['dog'], 'ComparisonOperator': 'EQ'}},
# missing 'p':
ExclusiveStartKey= { 'c': 'dog', 'x': 'cat' })
# When Query'ing on a certain GSI partition key and/or sort key,
# ExclusiveStartKey must have the same partition/sort key value.
# Reproduces issue #26988.
@pytest.mark.xfail(reason="issue #26988")
def test_gsi_query_exclusivestartkey_wrong_partition(test_table_gsi_5):
# The error that DynamoDB reports if the wrong partition is mentioned
# in ExclusiveStartKey is "The provided starting key is outside query
# boundaries based on provided conditions".
# Query a whole GSI partition:
with pytest.raises(ClientError, match='ValidationException.*starting key'):
test_table_gsi_5.query(
ConsistentRead=False,
IndexName='hello',
KeyConditions={'p': { 'AttributeValueList': ['right'], 'ComparisonOperator': 'EQ'}},
ExclusiveStartKey= { 'p': 'wrong', 'c': 'dog', 'x': 'cat' })
# Query with a GSI partition and sort key (this can still return
# multiple items, so ExclusiveStartKey makes sense).
with pytest.raises(ClientError, match='ValidationException.*starting key'):
test_table_gsi_5.query(
ConsistentRead=False,
IndexName='hello',
KeyConditions={
'p': { 'AttributeValueList': ['rightp'], 'ComparisonOperator': 'EQ'},
'x': { 'AttributeValueList': ['rightx'], 'ComparisonOperator': 'EQ'}
},
ExclusiveStartKey= { 'p': 'rightp', 'c': 'dog', 'x': 'wrongx' })
# Confirm that rightp, rightx the ExclusiveStartKey *is* supported
test_table_gsi_5.query(
ConsistentRead=False,
IndexName='hello',
KeyConditions={
'p': { 'AttributeValueList': ['rightp'], 'ComparisonOperator': 'EQ'},
'x': { 'AttributeValueList': ['rightx'], 'ComparisonOperator': 'EQ'}
},
ExclusiveStartKey= { 'p': 'rightp', 'c': 'dog', 'x': 'rightx' })
# If ExclusiveStartKey's p includes the wrong value but x is right,
# DynamoDB prints a different - and strange - error message: "The query
# can return at most one row and cannot be restarted". This message is not
# really true: As we saw in the previous statement, when ExclusiveStartKey
# does contain the right values - it is allowed, and this query doesn't
# really return just one row.
with pytest.raises(ClientError, match='ValidationException'):
test_table_gsi_5.query(
ConsistentRead=False,
IndexName='hello',
KeyConditions={
'p': { 'AttributeValueList': ['rightp'], 'ComparisonOperator': 'EQ'},
'x': { 'AttributeValueList': ['rightx'], 'ComparisonOperator': 'EQ'}
},
ExclusiveStartKey= { 'p': 'wrongp', 'c': 'dog', 'x': 'rightx' })
# Check that ExclusiveStartKey cannot contain any spurious column names
# beyond the actual primary key columns of the base and the GSI.
# Reproduces issue #26988.
@pytest.mark.xfail(reason="issue #26988")
def test_gsi_query_exclusivestartkey_spurious_column(test_table_gsi_5):
query_args = {
'ConsistentRead': False,
'IndexName': 'hello',
'KeyConditions': {
'p': { 'AttributeValueList': ['dog'], 'ComparisonOperator': 'EQ'},
'x': { 'AttributeValueList': ['cat'], 'ComparisonOperator': 'EQ'}
}
}
exclusive_start_key = { 'p': 'dog', 'c': 'mouse', 'x': 'cat' }
# This ExclusiveStartKey has the right columns - p, c and x - and works.
test_table_gsi_5.query(ExclusiveStartKey=exclusive_start_key, **query_args)
# If we add to exclusive_start_key a spurious column y, it fails.
exclusive_start_key['y'] = 'meerkat'
with pytest.raises(ClientError, match='ValidationException.*starting key'):
test_table_gsi_5.query(ExclusiveStartKey=exclusive_start_key, **query_args)
Reference in New Issue View Git Blame Copy Permalink