mirror of
https://github.com/scylladb/scylladb.git
synced 2026-06-06 15:03:06 +00:00
13ef31f38b
In the DynamoDB API, expressions (e.g., ConditionExpression and many more)
may contain references to column names ("#name") or to values (":val")
given in a separate part of the request - ExpressionAttributeNames and
ExpressionAttributeValues respectively.
Before this patch, we resolved these references as part of the expression's
evaluation. This approach had two downsides:
1. It often misdiagnosed (both false negatives and false positives) cases
of unused names and values in expressions. We already had two xfailing
tests with examples - which pass after this patch. This patch also
adds two additional tests, which failed before this patch and pass
with it.
2. In one of the following patches we will add support for FilterExpression,
where the same expression is used repeatedly on many items. It is a waste
(as well as makes the code uglier) to resolve the same references again
and again each time the expression is evaluated. We should be able
to do it just once.
So this patch introduces an intermediate step between parsing and evaluating
an expression - "resolving" the expression. The new resolve_*() functions
modify the already parsed expression, replacing references to attribute
names and constant values by the actual names and values taken from the
request. The resolve_*() functions also keep track which references were
used, making it very easy to check (as DynamoDB does) if there are any
unused names or values, before starting the evaluation.
The interface of evaluate() functions become much simpler - they no longer
need to know the original request (which was previously needed for
ExpressionAttributeNames/Values), the table's schema (which was previously
needed only for some error checking), keep track of which references were
used. This simplification is helpful for using the expressions in contexts
where these things (request and schema) are no longer conveniently available,
namely in FilterExpression.
A small side-benefit of this patch is that it moves a bit of code, which
handled resolving of references in expressions, from executor.cc to
expressions.cc. This is just the first step in a bigger effort to
reduce the size of executor.cc by moving code to smaller source files.
There is no attempt in this patch to move as much code as we can.
We will move more code in a separate patch in this series.
Fixes #6572.
Signed-off-by: Nadav Har'El <nyh@scylladb.com>
218 lines
14 KiB
Python
218 lines
14 KiB
Python
# Copyright 2019 ScyllaDB
|
|
#
|
|
# This file is part of Scylla.
|
|
#
|
|
# Scylla is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Scylla is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU Affero General Public License
|
|
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
# Tests for the various operations (GetItem, Query, Scan) with a
|
|
# ProjectionExpression parameter.
|
|
#
|
|
# ProjectionExpression is an expension of the legacy AttributesToGet
|
|
# parameter. Both parameters request that only a subset of the attributes
|
|
# be fetched for each item, instead of all of them. But while AttributesToGet
|
|
# was limited to top-level attributes, ProjectionExpression can request also
|
|
# nested attributes.
|
|
|
|
import pytest
|
|
from botocore.exceptions import ClientError
|
|
from util import random_string, full_scan, full_query, multiset
|
|
|
|
# Basic test for ProjectionExpression, requesting only top-level attributes.
|
|
# Result should include the selected attributes only - if one wants the key
|
|
# attributes as well, one needs to select them explicitly. When no key
|
|
# attributes are selected, an item may have *none* of the selected
|
|
# attributes, and returned as an empty item.
|
|
def test_projection_expression_toplevel(test_table):
|
|
p = random_string()
|
|
c = random_string()
|
|
item = {'p': p, 'c': c, 'a': 'hello', 'b': 'hi'}
|
|
test_table.put_item(Item=item)
|
|
for wanted in [ ['a'], # only non-key attribute
|
|
['c', 'a'], # a key attribute (sort key) and non-key
|
|
['p', 'c'], # entire key
|
|
['nonexistent'] # Our item doesn't have this
|
|
]:
|
|
got_item = test_table.get_item(Key={'p': p, 'c': c}, ProjectionExpression=",".join(wanted), ConsistentRead=True)['Item']
|
|
expected_item = {k: item[k] for k in wanted if k in item}
|
|
assert expected_item == got_item
|
|
|
|
# Various simple tests for ProjectionExpression's syntax, using only top-evel
|
|
# attributes.
|
|
def test_projection_expression_toplevel_syntax(test_table_s):
|
|
p = random_string()
|
|
test_table_s.put_item(Item={'p': p, 'a': 'hello', 'b': 'hi'})
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a')['Item'] == {'a': 'hello'}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='#name', ExpressionAttributeNames={'#name': 'a'})['Item'] == {'a': 'hello'}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a,b')['Item'] == {'a': 'hello', 'b': 'hi'}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression=' a , b ')['Item'] == {'a': 'hello', 'b': 'hi'}
|
|
# Missing or unused names in ExpressionAttributeNames are errors:
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='#name', ExpressionAttributeNames={'#wrong': 'a'})['Item'] == {'a': 'hello'}
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='#name', ExpressionAttributeNames={'#name': 'a', '#unused': 'b'})['Item'] == {'a': 'hello'}
|
|
# It is not allowed to fetch the same top-level attribute twice (or in
|
|
# general, list two overlapping attributes). We get an error like
|
|
# "Invalid ProjectionExpression: Two document paths overlap with each
|
|
# other; must remove or rewrite one of these paths; path one: [a], path
|
|
# two: [a]".
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a,a')['Item']
|
|
# A comma with nothing after it is a syntax error:
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a,')['Item']
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression=',a')['Item']
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a,,b')['Item']
|
|
# An empty ProjectionExpression is not allowed. DynamoDB recognizes its
|
|
# syntax, but then writes: "Invalid ProjectionExpression: The expression
|
|
# can not be empty".
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='')['Item']
|
|
|
|
# The following two tests are similar to test_projection_expression_toplevel()
|
|
# which tested the GetItem operation - but these test Scan and Query.
|
|
# Both test ProjectionExpression with only top-level attributes.
|
|
def test_projection_expression_scan(filled_test_table):
|
|
table, items = filled_test_table
|
|
for wanted in [ ['another'], # only non-key attributes (one item doesn't have it!)
|
|
['c', 'another'], # a key attribute (sort key) and non-key
|
|
['p', 'c'], # entire key
|
|
['nonexistent'] # none of the items have this attribute!
|
|
]:
|
|
got_items = full_scan(table, ProjectionExpression=",".join(wanted))
|
|
expected_items = [{k: x[k] for k in wanted if k in x} for x in items]
|
|
assert multiset(expected_items) == multiset(got_items)
|
|
|
|
def test_projection_expression_query(test_table):
|
|
p = random_string()
|
|
items = [{'p': p, 'c': str(i), 'a': str(i*10), 'b': str(i*100) } for i in range(10)]
|
|
with test_table.batch_writer() as batch:
|
|
for item in items:
|
|
batch.put_item(item)
|
|
for wanted in [ ['a'], # only non-key attributes
|
|
['c', 'a'], # a key attribute (sort key) and non-key
|
|
['p', 'c'], # entire key
|
|
['nonexistent'] # none of the items have this attribute!
|
|
]:
|
|
got_items = full_query(test_table, KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}}, ProjectionExpression=",".join(wanted))
|
|
expected_items = [{k: x[k] for k in wanted if k in x} for x in items]
|
|
assert multiset(expected_items) == multiset(got_items)
|
|
|
|
# The previous tests all fetched only top-level attributes. They could all
|
|
# be written using AttributesToGet instead of ProjectionExpression (and,
|
|
# in fact, we do have similar tests with AttributesToGet in other files),
|
|
# but the previous test checked that the alternative syntax works correctly.
|
|
# The following test checks fetching more elaborate attribute paths from
|
|
# nested documents.
|
|
@pytest.mark.xfail(reason="ProjectionExpression does not yet support attribute paths")
|
|
def test_projection_expression_path(test_table_s):
|
|
p = random_string()
|
|
test_table_s.put_item(Item={
|
|
'p': p,
|
|
'a': {'b': [2, 4, {'x': 'hi', 'y': 'yo'}], 'c': 5},
|
|
'b': 'hello'
|
|
})
|
|
# Fetching the entire nested document "a" works, of course:
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a')['Item'] == {'a': {'b': [2, 4, {'x': 'hi', 'y': 'yo'}], 'c': 5}}
|
|
# If we fetch a.b, we get only the content of b - but it's still inside
|
|
# the a dictionary:
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.b')['Item'] == {'a': {'b': [2, 4, {'x': 'hi', 'y': 'yo'}]}}
|
|
# Similarly, fetching a.b[0] gives us a one-element array in a dictionary.
|
|
# Note that [0] is the first element of an array.
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.b[0]')['Item'] == {'a': {'b': [2]}}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.b[2]')['Item'] == {'a': {'b': [{'x': 'hi', 'y': 'yo'}]}}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.b[2].y')['Item'] == {'a': {'b': [{'y': 'yo'}]}}
|
|
# Trying to read any sort of non-existent attribute returns an empty item.
|
|
# This includes a non-existing top-level attribute, an attempt to read
|
|
# beyond the end of an array or a non-existent member of a dictionary, as
|
|
# well as paths which begin with a non-existent prefix.
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='x')['Item'] == {}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.b[3]')['Item'] == {}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.x')['Item'] == {}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.x.y')['Item'] == {}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.b[3].x')['Item'] == {}
|
|
# We can read multiple paths - the result are merged into one object
|
|
# structured the same was as in the original item:
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.b[0],a.b[1]')['Item'] == {'a': {'b': [2, 4]}}
|
|
assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.b[0],a.c')['Item'] == {'a': {'b': [2], 'c': 5}}
|
|
# It is not allowed to read the same path multiple times. The error from
|
|
# DynamoDB looks like: "Invalid ProjectionExpression: Two document paths
|
|
# overlap with each other; must remove or rewrite one of these paths;
|
|
# path one: [a, b, [0]], path two: [a, b, [0]]".
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a.b[0],a.b[0]')['Item']
|
|
# Two paths are considered to "overlap" if the content of one path
|
|
# contains the content of the second path. So requesting both "a" and
|
|
# "a.b[0]" is not allowed.
|
|
with pytest.raises(ClientError, match='ValidationException'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a,a.b[0]')['Item']
|
|
|
|
@pytest.mark.xfail(reason="ProjectionExpression does not yet support attribute paths")
|
|
def test_query_projection_expression_path(test_table):
|
|
p = random_string()
|
|
items = [{'p': p, 'c': str(i), 'a': {'x': str(i*10), 'y': 'hi'}, 'b': 'hello' } for i in range(10)]
|
|
with test_table.batch_writer() as batch:
|
|
for item in items:
|
|
batch.put_item(item)
|
|
got_items = full_query(test_table, KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}}, ProjectionExpression="a.x")
|
|
expected_items = [{'a': {'x': x['a']['x']}} for x in items]
|
|
assert multiset(expected_items) == multiset(got_items)
|
|
|
|
@pytest.mark.xfail(reason="ProjectionExpression does not yet support attribute paths")
|
|
def test_scan_projection_expression_path(test_table):
|
|
# This test is similar to test_query_projection_expression_path above,
|
|
# but uses a scan instead of a query. The scan will generate unrelated
|
|
# partitions created by other tests (hopefully not too many...) that we
|
|
# need to ignore. We also need to ask for "p" too, so we can filter by it.
|
|
p = random_string()
|
|
items = [{'p': p, 'c': str(i), 'a': {'x': str(i*10), 'y': 'hi'}, 'b': 'hello' } for i in range(10)]
|
|
with test_table.batch_writer() as batch:
|
|
for item in items:
|
|
batch.put_item(item)
|
|
got_items = [ x for x in full_scan(test_table, ProjectionExpression="p, a.x") if x['p'] == p]
|
|
expected_items = [{'p': p, 'a': {'x': x['a']['x']}} for x in items]
|
|
assert multiset(expected_items) == multiset(got_items)
|
|
|
|
# It is not allowed to use both ProjectionExpression and its older cousin,
|
|
# AttributesToGet, together. If trying to do this, DynamoDB produces an error
|
|
# like "Can not use both expression and non-expression parameters in the same
|
|
# request: Non-expression parameters: {AttributesToGet} Expression
|
|
# parameters: {ProjectionExpression}
|
|
def test_projection_expression_and_attributes_to_get(test_table_s):
|
|
p = random_string()
|
|
test_table_s.put_item(Item={'p': p, 'a': 'hello', 'b': 'hi'})
|
|
with pytest.raises(ClientError, match='ValidationException.*both'):
|
|
test_table_s.get_item(Key={'p': p}, ConsistentRead=True, ProjectionExpression='a', AttributesToGet=['b'])['Item']
|
|
with pytest.raises(ClientError, match='ValidationException.*both'):
|
|
full_scan(test_table_s, ProjectionExpression='a', AttributesToGet=['a'])
|
|
with pytest.raises(ClientError, match='ValidationException.*both'):
|
|
full_query(test_table_s, KeyConditions={'p': {'AttributeValueList': [p], 'ComparisonOperator': 'EQ'}}, ProjectionExpression='a', AttributesToGet=['a'])
|
|
|
|
# above in test_projection_expression_toplevel_syntax among other things
|
|
# we noted how spurious entries in ExpressionAttributeNames, not needed
|
|
# the the ProjectionExpression, cause an error. Sometimes we have two
|
|
# expressions in the same request, for example, both a ProjectionExpression
|
|
# and a KeyConditionExpression. It's only an error if a name is not
|
|
# needed by both of these expressions
|
|
def test_projection_expression_and_key_condition_expression(test_table_s):
|
|
p = random_string()
|
|
test_table_s.put_item(Item={'p': p, 'a': 'hello', 'b': 'hi'})
|
|
got_items = full_query(test_table_s,
|
|
KeyConditionExpression='#name1 = :val1',
|
|
ProjectionExpression='#name2',
|
|
ExpressionAttributeNames={'#name1': 'p', '#name2': 'a'},
|
|
ExpressionAttributeValues={':val1': p});
|
|
assert got_items == [{'a': 'hello'}]
|