# Copyright 2019-present ScyllaDB # # SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0 # Tests for the UpdateItem operations with an UpdateExpression parameter import pytest from botocore.exceptions import ClientError from test.alternator.util import random_string # The simplest test of using UpdateExpression to set a top-level attribute, # instead of the older AttributeUpdates parameter. # Checks only one "SET" action in an UpdateExpression. def test_update_expression_set(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1', ExpressionAttributeValues={':val1': 4}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 4} # An empty UpdateExpression is NOT allowed, and generates a "The expression # can not be empty" error. This contrasts with an empty AttributeUpdates which # is allowed, and results in the creation of an empty item if it didn't exist # yet (see test_empty_update()). def test_update_expression_empty(test_table_s): p = random_string() with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='') # A basic test with multiple SET actions in one expression def test_update_expression_set_multi(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='SET x = :val1, y = :val1', ExpressionAttributeValues={':val1': 4}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'x': 4, 'y': 4} # Test that UpdateItem merges the change with a previously existing item - # it doesn't outright replace it like PutItem does. This merge is done # even if the expression doesn't need to read the old value of the item. def test_update_expression_set_merge(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': 'hi'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hi'} test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val', ExpressionAttributeValues={':val': 'hello'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hi', 'b': 'hello'} test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :val', ExpressionAttributeValues={':val': 'hey'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hey', 'b': 'hello'} # SET can be used to copy an existing attribute to a new one def test_update_expression_set_copy(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': 'hello'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hello'} test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = a') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hello', 'b': 'hello'} # Copying an non-existing attribute generates an error with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET c = z') # Same thing happens if the item doesn't exist at all p1 = random_string() with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p1}, UpdateExpression='SET c = z') # It turns out that attributes to be copied are read before the SET # starts to write, so "SET x = :val1, y = x" does not work... with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET x = :val1, y = x', ExpressionAttributeValues={':val1': 4}) # SET z=z does nothing if z exists, or fails if it doesn't test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = a') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hello', 'b': 'hello'} with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET z = z') # We can also use name references in either LHS or RHS of SET, e.g., # SET #one = #two. We need to also take the references used in the RHS # when we want to complain about unused names in ExpressionAttributeNames. test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #one = #two', ExpressionAttributeNames={'#one': 'c', '#two': 'a'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hello', 'b': 'hello', 'c': 'hello'} with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #one = #two', ExpressionAttributeNames={'#one': 'c', '#two': 'a', '#three': 'z'}) # Test for read-before-write action where the value to be read is nested inside a - operator def test_update_expression_set_nested_copy(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #n = :two', ExpressionAttributeNames={'#n': 'n'}, ExpressionAttributeValues={':two': 2}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #nn = :seven - #n', ExpressionAttributeNames={'#nn': 'nn', '#n': 'n'}, ExpressionAttributeValues={':seven': 7}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'n': 2, 'nn': 5} test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #nnn = :nnn', ExpressionAttributeNames={'#nnn': 'nnn'}, ExpressionAttributeValues={':nnn': [2,4]}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #nnnn = list_append(:val1, #nnn)', ExpressionAttributeNames={'#nnnn': 'nnnn', '#nnn': 'nnn'}, ExpressionAttributeValues={':val1': [1,3]}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'n': 2, 'nn': 5, 'nnn': [2,4], 'nnnn': [1,3,2,4]} # Test for getting a key value with read-before-write def test_update_expression_set_key(test_table_sn): p = random_string() test_table_sn.update_item(Key={'p': p, 'c': 7}) test_table_sn.update_item(Key={'p': p, 'c': 7}, UpdateExpression='SET #n = #p', ExpressionAttributeNames={'#n': 'n', '#p': 'p'}) test_table_sn.update_item(Key={'p': p, 'c': 7}, UpdateExpression='SET #nn = #c + #c', ExpressionAttributeNames={'#nn': 'nn', '#c': 'c'}) assert test_table_sn.get_item(Key={'p': p, 'c': 7}, ConsistentRead=True)['Item'] == {'p': p, 'c': 7, 'n': p, 'nn': 14} # Simple test for the "REMOVE" action def test_update_expression_remove(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': 'hello', 'b': 'hi'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hello', 'b': 'hi'} test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 'hi'} # Demonstrate that although all DynamoDB examples give UpdateExpression # action names in uppercase - e.g., "SET", it can actually be any case. def test_update_expression_action_case(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1', ExpressionAttributeValues={':val1': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 3} test_table_s.update_item(Key={'p': p}, UpdateExpression='set b = :val1', ExpressionAttributeValues={':val1': 4}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 4} test_table_s.update_item(Key={'p': p}, UpdateExpression='sEt b = :val1', ExpressionAttributeValues={':val1': 5}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 5} # Demonstrate that whitespace is ignored in UpdateExpression parsing. def test_update_expression_action_whitespace(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='set b = :val1', ExpressionAttributeValues={':val1': 4}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 4} test_table_s.update_item(Key={'p': p}, UpdateExpression=' set b=:val1 ', ExpressionAttributeValues={':val1': 5}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 5} # In UpdateExpression, the attribute name can appear directly in the expression # (without a "#placeholder" notation) only if it is a single "token" as # determined by DynamoDB's lexical analyzer rules: Such token is composed of # alphanumeric characters whose first character must be alphabetic. Other # names cause the parser to see multiple tokens, and produce syntax errors. def test_update_expression_name_token(test_table_s): p = random_string() # Alphanumeric names starting with an alphabetical character work test_table_s.update_item(Key={'p': p}, UpdateExpression='SET alnum = :val1', ExpressionAttributeValues={':val1': 1}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['alnum'] == 1 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET Alpha_Numeric_123 = :val1', ExpressionAttributeValues={':val1': 2}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['Alpha_Numeric_123'] == 2 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET A123_ = :val1', ExpressionAttributeValues={':val1': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['A123_'] == 3 # But alphanumeric names cannot start with underscore or digits. # DynamoDB's lexical analyzer doesn't recognize them, and produces # a ValidationException looking like: # Invalid UpdateExpression: Syntax error; token: "_", near: "SET _123" with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET _123 = :val1', ExpressionAttributeValues={':val1': 3}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET _abc = :val1', ExpressionAttributeValues={':val1': 3}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET 123a = :val1', ExpressionAttributeValues={':val1': 3}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET 123 = :val1', ExpressionAttributeValues={':val1': 3}) # Various other non-alpha-numeric characters, split a token and NOT allowed with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET hi-there = :val1', ExpressionAttributeValues={':val1': 3}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET hi$there = :val1', ExpressionAttributeValues={':val1': 3}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET "hithere" = :val1', ExpressionAttributeValues={':val1': 3}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET !hithere = :val1', ExpressionAttributeValues={':val1': 3}) # In addition to the literal names, DynamoDB also allows references to any # name, using the "#reference" syntax. It turns out the reference name is # also a token following the rules as above, with one interesting point: # since "#" already started the token, the next character may be any # alphanumeric and doesn't need to be only alphabetical. # Note that the reference target - the actual attribute name - can include # absolutely any characters, and we use silly_name below as an example silly_name = '3can include any character!.#=' test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #Alpha_Numeric_123 = :val1', ExpressionAttributeValues={':val1': 4}, ExpressionAttributeNames={'#Alpha_Numeric_123': silly_name}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'][silly_name] == 4 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #123a = :val1', ExpressionAttributeValues={':val1': 5}, ExpressionAttributeNames={'#123a': silly_name}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'][silly_name] == 5 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #123 = :val1', ExpressionAttributeValues={':val1': 6}, ExpressionAttributeNames={'#123': silly_name}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'][silly_name] == 6 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #_ = :val1', ExpressionAttributeValues={':val1': 7}, ExpressionAttributeNames={'#_': silly_name}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'][silly_name] == 7 with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #hi-there = :val1', ExpressionAttributeValues={':val1': 7}, ExpressionAttributeNames={'#hi-there': silly_name}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #!hi = :val1', ExpressionAttributeValues={':val1': 7}, ExpressionAttributeNames={'#!hi': silly_name}) # Just a "#" is not enough as a token. Interestingly, DynamoDB will # find the bad name in ExpressionAttributeNames before it actually tries # to parse UpdateExpression, but we can verify the parse fails too by # using a valid but irrelevant name in ExpressionAttributeNames: with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET # = :val1', ExpressionAttributeValues={':val1': 7}, ExpressionAttributeNames={'#': silly_name}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET # = :val1', ExpressionAttributeValues={':val1': 7}, ExpressionAttributeNames={'#a': silly_name}) # There is also the value references, ":reference", for the right-hand # side of an assignment. These have similar naming rules like "#reference". test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :Alpha_Numeric_123', ExpressionAttributeValues={':Alpha_Numeric_123': 8}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 8 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :123a', ExpressionAttributeValues={':123a': 9}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 9 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :123', ExpressionAttributeValues={':123': 10}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 10 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :_', ExpressionAttributeValues={':_': 11}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 11 with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :hi!there', ExpressionAttributeValues={':hi!there': 12}) # Just a ":" is not enough as a token. with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :', ExpressionAttributeValues={':': 7}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :', ExpressionAttributeValues={':a': 7}) # Trying to use a :reference on the left-hand side of an assignment will # not work. In DynamoDB, it's a different type of token (and generates # syntax error). with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET :a = :b', ExpressionAttributeValues={':a': 1, ':b': 2}) # Multiple actions are allowed in one expression, but actions are divided # into clauses (SET, REMOVE, DELETE, ADD) and each of those can only appear # once. def test_update_expression_multi(test_table_s): p = random_string() # We can have two SET actions in one SET clause: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :val1, b = :val2', ExpressionAttributeValues={':val1': 1, ':val2': 2}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 1, 'b': 2} # But not two SET clauses - we get error "The "SET" section can only be used once in an update expression" with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :val1 SET b = :val2', ExpressionAttributeValues={':val1': 1, ':val2': 2}) # We can have a REMOVE and a SET clause (note no comma between clauses): test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a SET b = :val2', ExpressionAttributeValues={':val2': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 3} test_table_s.update_item(Key={'p': p}, UpdateExpression='SET c = :val2 REMOVE b', ExpressionAttributeValues={':val2': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'c': 3} # The same clause (e.g., SET) cannot be used twice, even if interleaved with something else with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :val1 REMOVE a SET b = :val2', ExpressionAttributeValues={':val1': 1, ':val2': 2}) # Trying to modify the same item twice in the same update is forbidden. # For "SET a=:v REMOVE a" DynamoDB says: "Invalid UpdateExpression: Two # document paths overlap with each other; must remove or rewrite one of # these paths; path one: [a], path two: [a]". # It is actually good for Scylla that such updates are forbidden, because had # we allowed "SET a=:v REMOVE a" the result would be surprising - because data # wins over a delete with the same timestamp, so "a" would be set despite the # REMOVE command appearing later in the command line. def test_update_expression_multi_overlap(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': 'hello'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hello'} # Neither "REMOVE a SET a = :v" nor "SET a = :v REMOVE a" are allowed: with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a SET a = :v', ExpressionAttributeValues={':v': 'hi'}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :v REMOVE a', ExpressionAttributeValues={':v': 'yo'}) # It's also not allowed to set a twice in the same clause with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :v1, a = :v2', ExpressionAttributeValues={':v1': 'yo', ':v2': 'he'}) # Obviously, the paths are compared after the name references are evaluated with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #a1 = :v1, #a2 = :v2', ExpressionAttributeValues={':v1': 'yo', ':v2': 'he'}, ExpressionAttributeNames={'#a1': 'a', '#a2': 'a'}) # The problem isn't just with identical paths - we can't modify two paths that # "overlap" in the sense that one is the ancestor of the other. def test_update_expression_multi_overlap_nested(test_table_s): p = random_string() # Note that the overlap checks happen before checking the actual content # of the item, so it doesn't matter that the item we want to modify # doesn't even exist or have the structure referenced by the paths below. for expr in ['SET a = :val1, a.b = :val2', 'SET a.b = :val1, a = :val2', 'SET a.b = :val1, a.b = :val2', 'SET a.b = :val1, a.b.c = :val2', 'SET a.b.c = :val1, a.b = :val2', 'SET a.b.c = :val1, a.b.c = :val2', 'SET a.b = :val1, a.b.c.d.e = :val2', 'SET a.b.c.d.e = :val1, a.b = :val2', 'SET a = :val1, a[1] = :val2', 'SET a[1] = :val1, a = :val2', 'SET a[1] = :val1, a[1] = :val2', 'SET a[1][1] = :val1, a[1] = :val2', 'SET a[1] = :val1, a[1][1] = :val2', 'SET a[1][1] = :val1, a[1][1] = :val2', 'SET a[1][1][1][1] = :val1, a[1][1] = :val2', 'SET a[1][1] = :val1, a[1][1][1][1] = :val2', 'SET a[1][1][1][1] = :val1, a[1][1][1][1] = :val2', ]: print(expr) with pytest.raises(ClientError, match='ValidationException.*overlap'): test_table_s.update_item(Key={'p': p}, UpdateExpression=expr, ExpressionAttributeValues={':val1': 2, ':val2': 'there'}) # Obviously this test can trivially pass if overlap checks wrongly labels # everything as an overlap. So the test test_update_expression_multi_nested # below is important - it confirms that we can do multiple modifications # to the same item when they do not overlap. # Besides the concept of "overlapping" paths tested above, DynamoDB also has # the concept of "conflicting" paths - e.g., attempting to set both a.b and # a[1] together doesn't make sense. def test_update_expression_multi_conflict_nested(test_table_s): p = random_string() for expr in ['SET a.b = :val1, a[1] = :val2', 'SET a.b.c = :val1, a.b[2] = :val2', ]: print(expr) with pytest.raises(ClientError, match='ValidationException.*conflict'): test_table_s.update_item(Key={'p': p}, UpdateExpression=expr, ExpressionAttributeValues={':val1': 2, ':val2': 'there'}) # We can do several non-overlapping modifications to the same top-level # attribute and to different top-level attributes in the same update # expression. def test_update_expression_multi_nested(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': {'x': 3, 'y': 4, 'c': {'y': 3}}, 'b': {'x': 1, 'y': 2}}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a.b = :val1, a.c.d = :val2, b.x = :val3 REMOVE a.x, b.y', ExpressionAttributeValues={':val1': 10, ':val2': 'dog', ':val3': 17}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == { 'p': p, 'a': {'y': 4, 'b': 10, 'c': {'y': 3, 'd': 'dog'}}, 'b': {'x': 17}} # In the previous test we saw that *modifying* the same item twice in the same # update is forbidden; But it is allowed to *read* an item in the same update # that also modifies it, and we check this here. def test_update_expression_multi_with_copy(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': 'hello'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 'hello'} # "REMOVE a SET b = a" works: as noted in test_update_expression_set_copy() # the value of 'a' is read before the actual REMOVE operation happens. test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a SET b = a') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 'hello'} test_table_s.update_item(Key={'p': p}, UpdateExpression='SET c = b REMOVE b') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'c': 'hello'} # Test case where a :val1 is referenced, without being defined def test_update_expression_set_missing_value(test_table_s): p = random_string() with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1', ExpressionAttributeValues={':val2': 4}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1') # It is forbidden for ExpressionAttributeValues to contain values not used # by the expression. DynamoDB produces an error like: "Value provided in # ExpressionAttributeValues unused in expressions: keys: {:val1}" def test_update_expression_spurious_value(test_table_s): p = random_string() with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :val1', ExpressionAttributeValues={':val1': 3, ':val2': 4}) # Test case where a #name is referenced, without being defined def test_update_expression_set_missing_name(test_table_s): p = random_string() with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #name = :val1', ExpressionAttributeValues={':val2': 4}, ExpressionAttributeNames={'#wrongname': 'hello'}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #name = :val1', ExpressionAttributeValues={':val2': 4}) # It is forbidden for ExpressionAttributeNames to contain names not used # by the expression. DynamoDB produces an error like: "Value provided in # ExpressionAttributeNames unused in expressions: keys: {#b}" def test_update_expression_spurious_name(test_table_s): p = random_string() with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #a = :val1', ExpressionAttributeNames={'#a': 'hello', '#b': 'hi'}, ExpressionAttributeValues={':val1': 3, ':val2': 4}) # Test that the key attributes (hash key or sort key) cannot be modified # by an update def test_update_expression_cannot_modify_key(test_table_ss): p = random_string() c = random_string() with pytest.raises(ClientError, match='ValidationException.*key'): test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='SET p = :val1', ExpressionAttributeValues={':val1': 4}) with pytest.raises(ClientError, match='ValidationException.*key'): test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='SET c = :val1', ExpressionAttributeValues={':val1': 4}) with pytest.raises(ClientError, match='ValidationException.*key'): test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='REMOVE p') with pytest.raises(ClientError, match='ValidationException.*key'): test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='REMOVE c') with pytest.raises(ClientError, match='ValidationException.*key'): test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='ADD p :val1', ExpressionAttributeValues={':val1': 4}) with pytest.raises(ClientError, match='ValidationException.*key'): test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='ADD c :val1', ExpressionAttributeValues={':val1': 4}) with pytest.raises(ClientError, match='ValidationException.*key'): test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='DELETE p :val1', ExpressionAttributeValues={':val1': set(['cat', 'mouse'])}) with pytest.raises(ClientError, match='ValidationException.*key'): test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='DELETE c :val1', ExpressionAttributeValues={':val1': set(['cat', 'mouse'])}) # As sanity check, verify we *can* modify a non-key column test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='SET a = :val1', ExpressionAttributeValues={':val1': 4}) assert test_table_ss.get_item(Key={'p': p, 'c': c}, ConsistentRead=True)['Item'] == {'p': p, 'c': c, 'a': 4} test_table_ss.update_item(Key={'p': p, 'c': c}, UpdateExpression='REMOVE a') assert test_table_ss.get_item(Key={'p': p, 'c': c}, ConsistentRead=True)['Item'] == {'p': p, 'c': c} # Test that trying to start an expression with some nonsense like HELLO # instead of SET, REMOVE, ADD or DELETE, fails. def test_update_expression_non_existent_clause(test_table_s): p = random_string() with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='HELLO b = :val1', ExpressionAttributeValues={':val1': 4}) # Test support for "SET a = :val1 + :val2", "SET a = :val1 - :val2" # Only exactly these combinations work - e.g., it's a syntax error to # try to add three. Trying to add a string fails. def test_update_expression_plus_basic(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1 + :val2', ExpressionAttributeValues={':val1': 4, ':val2': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 7} test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1 - :val2', ExpressionAttributeValues={':val1': 5, ':val2': 2}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 3} # Only the addition of exactly two values is supported! with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1 + :val2 + :val3', ExpressionAttributeValues={':val1': 4, ':val2': 3, ':val3': 2}) # Only numeric values can be added - other things like strings or lists # cannot be added, and we get an error like "Incorrect operand type for # operator or function; operator or function: +, operand type: S". with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1 + :val2', ExpressionAttributeValues={':val1': 'dog', ':val2': 3}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1 + :val2', ExpressionAttributeValues={':val1': ['a', 'b'], ':val2': ['1', '2']}) # Test support for "SET a = b + :val2" et al., i.e., a version of the # above test_update_expression_plus_basic with read before write. def test_update_expression_plus_rmw(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': 2}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 2 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = a + :val1', ExpressionAttributeValues={':val1': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 5 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :val1 + a', ExpressionAttributeValues={':val1': 4}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 9 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = :val1 + a', ExpressionAttributeValues={':val1': 1}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == 10 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = b + a') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 19 # Test the list_append() function in SET, for the most basic use case of # concatenating two value references. Because this is the first test of # functions in SET, we also test some generic features of how functions # are parsed. def test_update_expression_list_append_basic(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(:val1, :val2)', ExpressionAttributeValues={':val1': [4, 'hello'], ':val2': ['hi', 7]}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': [4, 'hello', 'hi', 7]} # Unlike the operation name "SET", function names are case-sensitive! with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = LIST_APPEND(:val1, :val2)', ExpressionAttributeValues={':val1': [4, 'hello'], ':val2': ['hi', 7]}) # As usual, spaces are ignored by the parser test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(:val1, :val2)', ExpressionAttributeValues={':val1': ['a'], ':val2': ['b']}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': ['a', 'b']} # The list_append function only allows two parameters. The parser can # correctly parse fewer or more, but then an error is generated: "Invalid # UpdateExpression: Incorrect number of operands for operator or function; # operator or function: list_append, number of operands: 1". with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(:val1)', ExpressionAttributeValues={':val1': ['a']}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(:val1, :val2, :val3)', ExpressionAttributeValues={':val1': [4, 'hello'], ':val2': [7], ':val3': ['a']}) # If list_append is used on value which isn't a list, we get # error: "Invalid UpdateExpression: Incorrect operand type for operator # or function; operator or function: list_append, operand type: S" with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(:val1, :val2)', ExpressionAttributeValues={':val1': [4, 'hello'], ':val2': 'hi'}) # Additional list_append() tests, also using attribute paths as parameters # (i.e., read-modify-write). def test_update_expression_list_append(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :val1', ExpressionAttributeValues={':val1': ['hi', 2]}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] ==['hi', 2] # Often, list_append is used to append items to a list attribute test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(a, :val1)', ExpressionAttributeValues={':val1': [4, 'hello']}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == ['hi', 2, 4, 'hello'] # But it can also be used to just concatenate in other ways: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(:val1, a)', ExpressionAttributeValues={':val1': ['dog']}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == ['dog', 'hi', 2, 4, 'hello'] test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = list_append(a, :val1)', ExpressionAttributeValues={':val1': ['cat']}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == ['dog', 'hi', 2, 4, 'hello', 'cat'] test_table_s.update_item(Key={'p': p}, UpdateExpression='SET c = list_append(a, b)') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['c'] == ['dog', 'hi', 2, 4, 'hello', 'dog', 'hi', 2, 4, 'hello', 'cat'] # As usual, #references are allowed instead of inline names: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #name1 = list_append(#name2,:val1)', ExpressionAttributeValues={':val1': [8]}, ExpressionAttributeNames={'#name1': 'a', '#name2': 'a'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == ['dog', 'hi', 2, 4, 'hello', 8] # A list_append() function requires both its arguments to be lists. If one # of them isn't, we get a ValidationException. This also includes the case # that one of the arguments is an attribute which doesn't exist, or the # entire item doesn't exist. # The specific error message returned by Alternator and DynamoDB are not # identical - e.g., when an attribute is missing DynamoDB says that ""The # provided expression refers to an attribute that does not exist in the item" # and Alternator says "list_append() given a non-list" - but the error # type, ValidationException, is the same. def test_update_expression_list_append_non_list_arguments(test_table_s): p = random_string() val1 = ['hi', 2] def try_list_append(args): test_table_s.update_item(Key={'p': p}, UpdateExpression=f'SET a = list_append({args})', ExpressionAttributeValues={':val1': val1}) # The item p doesn't exist at all, so in particular the attribute "b" is # missing, so in particular b is not a list for args in ['b, :val1', ':val1, b']: with pytest.raises(ClientError, match='ValidationException'): try_list_append(args) # The item p does exist, but it doesn't have an attribute b, so # in particular b is not a list: test_table_s.put_item(Item={'p': p, 'a': 2}) for args in ['b, :val1', ':val1, b']: with pytest.raises(ClientError, match='ValidationException'): try_list_append(args) # Attribute b does exist, but it's not a list: for b in [7, 'hello', b'hi', set([1,2,3]), {'a': 3, 'b': 4}]: test_table_s.put_item(Item={'p': p, 'b': b}) for args in ['b, :val1', ':val1, b']: with pytest.raises(ClientError, match='ValidationException'): try_list_append(args) # Sanity check: when b does exist and *is* a list, things finally work. # Let's try a few lists, including an empty one and list including # nested list items - they are all fine: for b in [[1,2,3], [], ['x', ['hi', 'hello']]]: test_table_s.put_item(Item={'p': p, 'b': b}) try_list_append('b, :val1') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == b + val1 try_list_append(':val1, b') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == val1 + b # Test the "if_not_exists" function in SET # The test also checks additional features of function-call parsing. def test_update_expression_if_not_exists(test_table_s): p = random_string() # Since attribute a doesn't exist, set it: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = if_not_exists(a, :val1)', ExpressionAttributeValues={':val1': 2}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 2 # Now the attribute does exist, so set does nothing: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = if_not_exists(a, :val1)', ExpressionAttributeValues={':val1': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 2 # if_not_exists can also be used to check one attribute and set another, # but note that if_not_exists(a, :val) means a's value if it exists, # otherwise :val! test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = if_not_exists(c, :val1)', ExpressionAttributeValues={':val1': 4}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == 4 assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 2 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = if_not_exists(c, :val1)', ExpressionAttributeValues={':val1': 5}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == 5 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = if_not_exists(a, :val1)', ExpressionAttributeValues={':val1': 6}) # note how because 'a' does exist, its value is copied, overwriting b's # value: assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == 2 # The parser expects function parameters to be value references, paths, # or nested call to functions. Other crap will cause syntax errors: with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = if_not_exists(non@sense, :val1)', ExpressionAttributeValues={':val1': 6}) # if_not_exists() requires that the first parameter be a path. However, # the parser doesn't know this, and allows for a function parameter # also a value reference or a function call. If try one of these other # things the parser succeeds, but we get a later error, looking like: # "Invalid UpdateExpression: Operator or function requires a document # path; operator or function: if_not_exists" with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = if_not_exists(if_not_exists(a, :val2), :val1)', ExpressionAttributeValues={':val1': 6, ':val2': 3}) with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = if_not_exists(:val2, :val1)', ExpressionAttributeValues={':val1': 6, ':val2': 3}) # Surprisingly, if the wrong argument is a :val value reference, the # parser first tries to look it up in ExpressionAttributeValues (and # fails if it's missing), before realizing any value reference would be # wrong... So the following fails like the above does - but with a # different error message (which we do not check here): "Invalid # UpdateExpression: An expression attribute value used in expression # is not defined; attribute value: :val2" with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = if_not_exists(:val2, :val1)', ExpressionAttributeValues={':val1': 6}) # When the expression parser parses a function call f(value, value), each # value may itself be a function call - ad infinitum. So expressions like # list_append(if_not_exists(a, :val1), :val2) are legal and so is deeper # nesting. @pytest.mark.xfail(reason="for unknown reason, DynamoDB does not allow nesting list_append") def test_update_expression_function_nesting(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(if_not_exists(a, :val1), :val2)', ExpressionAttributeValues={':val1': ['a', 'b'], ':val2': ['cat', 'dog']}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == ['a', 'b', 'cat', 'dog'] test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(if_not_exists(a, :val1), :val2)', ExpressionAttributeValues={':val1': ['a', 'b'], ':val2': ['1', '2']}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == ['a', 'b', 'cat', 'dog', '1', '2'] # It's even fine to nest two calls of the same if_not_exists function: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = if_not_exists(b, if_not_exists(c, :val1))', ExpressionAttributeValues={':val1': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 3 test_table_s.update_item(Key={'p': p}, UpdateExpression='SET x = if_not_exists(b, if_not_exists(a, :val1))', ExpressionAttributeValues={':val1': 4}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['x'] == 3 # I don't understand why the following expression isn't accepted, but it # isn't! It produces a "Invalid UpdateExpression: The function is not # allowed to be used this way in an expression; function: list_append". # I don't know how to explain it. In any case, the *parsing* works - # this is not a syntax error - the failure is in some verification later. with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(list_append(:val1, :val2), :val3)', ExpressionAttributeValues={':val1': ['a'], ':val2': ['1'], ':val3': ['hi']}) # Ditto, the following passes the parser but fails some later check with # the same error message as above. with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = list_append(list_append(list_append(:val1, :val2), :val3), :val4)', ExpressionAttributeValues={':val1': ['a'], ':val2': ['1'], ':val3': ['hi'], ':val4': ['yo']}) # Verify how in SET expressions, "+" (or "-") nests with functions. # We discover that f(x)+f(y) works but f(x+y) does NOT (results in a syntax # error on the "+"). This means that the parser has two separate rules: # 1. set_action: SET path = value + value # 2. value: VALREF | NAME | NAME (value, ...) def test_update_expression_function_plus_nesting(test_table_s): p = random_string() # As explained above, this - with "+" outside the expression, works: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = if_not_exists(b, :val1)+:val2', ExpressionAttributeValues={':val1': 2, ':val2': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == 5 # ...but this - with the "+" inside an expression parameter, is a syntax # error: with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET c = if_not_exists(c, :val1+:val2)', ExpressionAttributeValues={':val1': 5, ':val2': 4}) # This test tries to use an undefined function "f". This, obviously, fails, # but where we to actually print the error we would see "Invalid # UpdateExpression: Invalid function name; function: f". Not a syntax error. # This means that the parser accepts any alphanumeric name as a function # name, and only later use of this function fails because it's not one of # the supported file. def test_update_expression_unknown_function(test_table_s): p = random_string() with pytest.raises(ClientError, match='ValidationException.*f'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = f(b,c,d)') with pytest.raises(ClientError, match='ValidationException.*f123_hi'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = f123_hi(b,c,d)') # Just like unreferenced column names parsed by the DynamoDB parser, # function names must also start with an alphabetic character. Trying # to use _f as a function name will result with an actual syntax error, # on the "_" token. with pytest.raises(ClientError, match='ValidationException.*yntax error'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = _f(b,c,d)') # Test "ADD" operation for numbers def test_update_expression_add_numbers(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': 3, 'b': 'hi'}) test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD a :val1', ExpressionAttributeValues={':val1': 4}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == 7 # If the value to be added isn't a number, we get an error like "Invalid # UpdateExpression: Incorrect operand type for operator or function; # operator: ADD, operand type: STRING". with pytest.raises(ClientError, match='ValidationException.*type'): test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD a :val1', ExpressionAttributeValues={':val1': 'hello'}) # Similarly, if the attribute we're adding to isn't a number, we get an # error like "An operand in the update expression has an incorrect data # type" with pytest.raises(ClientError, match='ValidationException.*type'): test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD b :val1', ExpressionAttributeValues={':val1': 1}) # In test_update_expression_add_numbers() above we tested ADDing a number to # an existing number. The following test check that ADD can be used to # create a *new* number, as if it was added to zero. def test_update_expression_add_numbers_new(test_table_s): # Test that "ADD" can create a new number attribute: p = random_string() test_table_s.put_item(Item={'p': p, 'a': 'hello'}) test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD b :val1', ExpressionAttributeValues={':val1': 7}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == 7 # Test that "ADD" can create an entirely new item: p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD b :val1', ExpressionAttributeValues={':val1': 8}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == 8 # Test "ADD" operation for sets def test_update_expression_add_sets(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': set(['dog', 'cat', 'mouse']), 'b': 'hi'}) test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD a :val1', ExpressionAttributeValues={':val1': set(['pig'])}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == set(['dog', 'cat', 'mouse', 'pig']) # TODO: right now this test won't detect duplicated values in the returned result, # because boto3 parses a set out of the returned JSON anyway. This check should leverage # lower level API (if exists) to ensure that the JSON contains no duplicates # in the set representation. It has been verified manually. test_table_s.put_item(Item={'p': p, 'a': set(['beaver', 'lynx', 'coati']), 'b': 'hi'}) test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD a :val1', ExpressionAttributeValues={':val1': set(['coati', 'beaver', 'badger'])}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == set(['beaver', 'badger', 'lynx', 'coati']) # The value to be added needs to be a set of the same type - it can't # be a single element or anything else. If the value has the wrong type, # we get an error like "Invalid UpdateExpression: Incorrect operand type # for operator or function; operator: ADD, operand type: STRING". with pytest.raises(ClientError, match='ValidationException.*type'): test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD a :val1', ExpressionAttributeValues={':val1': 'hello'}) # In test_update_expression_add_sets() above we tested ADDing elements to an # existing set. The following test checks that ADD can be used to create a # *new* set, by adding its first item. def test_update_expression_add_sets_new(test_table_s): # Test that "ADD" can create a new set attribute: p = random_string() test_table_s.put_item(Item={'p': p, 'a': 'hello'}) test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD b :val1', ExpressionAttributeValues={':val1': set(['dog'])}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == set(['dog']) # Test that "ADD" can create an entirely new item: p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD b :val1', ExpressionAttributeValues={':val1': set(['cat'])}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['b'] == set(['cat']) # Although AttributeUpdate's ADD operation works on lists (despite being # documented only for sets; see test_item.py::test_update_item_add), the # ADD operation of UpdateExpression does *not* support lists. Let's verify # this here. # Passing this test will require the two ADD implementations to be slightly # different. def test_update_expression_add_lists(test_table_s): p = random_string() # When the operand given in ExpressionAttributesValues is a list, # we get an error about the operand, no matter what the item contains - # a list too, or a set. test_table_s.put_item(Item={'p': p, 'a': [1, 2]}) with pytest.raises(ClientError, match='ValidationException.*operand'): test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD a :val1', ExpressionAttributeValues={':val1': [3, 4]}) test_table_s.put_item(Item={'p': p, 'a': set([1, 2])}) with pytest.raises(ClientError, match='ValidationException.*operand'): test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD a :val1', ExpressionAttributeValues={':val1': [3, 4]}) # If the operand is a set, the item value still cannot be a list because # those different types can't be added. with pytest.raises(ClientError, match='ValidationException.*operand'): test_table_s.put_item(Item={'p': p, 'a': [1, 2]}) test_table_s.update_item(Key={'p': p}, UpdateExpression='ADD a :val1', ExpressionAttributeValues={':val1': set([3, 4])}) # Test "DELETE" operation for sets def test_update_expression_delete_sets(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': set(['dog', 'cat', 'mouse']), 'b': 'hi'}) test_table_s.update_item(Key={'p': p}, UpdateExpression='DELETE a :val1', ExpressionAttributeValues={':val1': set(['cat', 'mouse'])}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == set(['dog']) # Deleting an element not present in the set is not an error - it just # does nothing test_table_s.update_item(Key={'p': p}, UpdateExpression='DELETE a :val1', ExpressionAttributeValues={':val1': set(['pig'])}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == set(['dog']) # Deleting all the elements cannot leave an empty set (which isn't # supported). Rather, it deletes the attribute altogether: test_table_s.update_item(Key={'p': p}, UpdateExpression='DELETE a :val1', ExpressionAttributeValues={':val1': set(['dog'])}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 'hi'} # Deleting elements from a non-existent attribute is allowed, and # simply does nothing: test_table_s.update_item(Key={'p': p}, UpdateExpression='DELETE a :val1', ExpressionAttributeValues={':val1': set(['dog'])}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'b': 'hi'} # An empty set parameter is not allowed with pytest.raises(ClientError, match='ValidationException.*empty'): test_table_s.update_item(Key={'p': p}, UpdateExpression='DELETE a :val1', ExpressionAttributeValues={':val1': set([])}) # The value to be deleted must be a set of the same type - it can't # be a single element or anything else. If the value has the wrong type, # we get an error like "Invalid UpdateExpression: Incorrect operand type # for operator or function; operator: DELETE, operand type: STRING". test_table_s.put_item(Item={'p': p, 'a': set(['dog', 'cat', 'mouse']), 'b': 'hi'}) with pytest.raises(ClientError, match='ValidationException.*type'): test_table_s.update_item(Key={'p': p}, UpdateExpression='DELETE a :val1', ExpressionAttributeValues={':val1': 'hello'}) ######## Tests for paths and nested attribute updates: # A dot inside a name in ExpressionAttributeNames is a literal dot, and # results in a top-level attribute with an actual dot in its name - not # a nested attribute path. def test_update_expression_dot_in_name(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='SET #a = :val1', ExpressionAttributeValues={':val1': 3}, ExpressionAttributeNames={'#a': 'a.b'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a.b': 3} # Below we have several tests of what happens when a nested attribute is # on the left-hand side of an assignment, but an every simpler case of # nested attributes is having one on the right hand side of an assignment: def test_update_expression_nested_attribute_rhs(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': {'b': 3, 'c': {'x': 7, 'y': 8}}, 'd': 5}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET z = a.c.x') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['z'] == 7 # A basic test for direct update of a nested attribute: One of the top-level # attributes is itself a document, and we update only one of that document's # nested attributes. def test_update_expression_nested_attribute_dot(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': {'b': 3, 'c': 4}, 'd': 5}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': {'b': 3, 'c': 4}, 'd': 5} test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a.c = :val1', ExpressionAttributeValues={':val1': 7}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': {'b': 3, 'c': 7}, 'd': 5} # Of course we can also add new nested attributes, not just modify # existing ones: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a.d = :val1', ExpressionAttributeValues={':val1': 3}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': {'b': 3, 'c': 7, 'd': 3}, 'd': 5} # Similar test, for a list: one of the top-level attributes is a list, we # can update one of its items. def test_update_expression_nested_attribute_index(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': ['one', 'two', 'three']}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a[1] = :val1', ExpressionAttributeValues={':val1': 'hello'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': ['one', 'hello', 'three']} # An index into a list must be an actual integer - DynamoDB does not support # a value reference (:xyz) to be used as a index. This is the same test as the # above test_update_expression_nested_attribute_index() - we just try to use # the a reference :xyz for the index 1. And it's considered a syntax error def test_update_expression_nested_attribute_index_reference(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': ['one', 'two', 'three']}) with pytest.raises(ClientError, match='ValidationException.*yntax'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a[:xyz] = :val1', ExpressionAttributeValues={':val1': 'hello', ':xyz': 1}) # In the previous test we saw that a value reference (:xyz) can't be used # as an index. Here we see that a name reference (#xyz) also can't be used # as an index. It too is a syntax error - it doesn't even matter if we # define this #xyz or not. def test_update_expression_nested_attribute_index_reference_name(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': ['one', 'two', 'three']}) with pytest.raises(ClientError, match='ValidationException.*yntax'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a[#xyz] = :val1', ExpressionAttributeValues={':val1': 'hello'}) # Test that just like happens in top-level attributes, also in nested # attributes, setting them replaces the old value - potentially an entire # nested document, by the whole value (which may have a different type) def test_update_expression_nested_different_type(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': {'b': 3, 'c': {'one': 1, 'two': 2}}}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a.c = :val1', ExpressionAttributeValues={':val1': 7}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': {'b': 3, 'c': 7}} # Yet another test of a nested attribute update. This one uses deeper # level of nesting (dots and indexes), adds #name references to the mix. def test_update_expression_nested_deep(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': {'b': 3, 'c': ['hi', {'x': {'y': [3, 5, 7]}}]}}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a.c[1].#name.y[1] = :val1', ExpressionAttributeValues={':val1': 9}, ExpressionAttributeNames={'#name': 'x'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == {'b': 3, 'c': ['hi', {'x': {'y': [3, 9, 7]}}]} # A deep path can also appear on the right-hand-side of an assignment test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a.z = a.c[1].#name.y[1]', ExpressionAttributeNames={'#name': 'x'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a']['z'] == 9 # A REMOVE operation can be used to remove nested attributes, and also # individual list items. def test_update_expression_nested_remove(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': {'b': 3, 'c': ['hi', {'x': {'y': [3, 5, 7]}, 'q': 2}]}}) test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a.c[1].x.y[1], a.c[1].q') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == {'b': 3, 'c': ['hi', {'x': {'y': [3, 7]}}]} # Removing a list item beyond the end of the list (e.g., REMOVE a[17] when # the list only has three items) is silently ignored. def test_update_expression_nested_remove_list_item_after_end(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': [4, 5, 6]}) test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a[17]') # If we remove a[1] and then change a[3], the index "3" refers to the position # *before* the first removal. def test_update_expression_nested_remove_list_item_original_number(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': [2, 3, 4, 5, 6, 7]}) test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a[1] SET a[3] = :val', ExpressionAttributeValues={':val': 17}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == [2, 4, 17, 6, 7] # The order of the operations doesn't matter test_table_s.put_item(Item={'p': p, 'a': [2, 3, 4, 5, 6, 7]}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a[3] = :val REMOVE a[1]', ExpressionAttributeValues={':val': 17}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == [2, 4, 17, 6, 7] # DynamoDB allows an empty map. So removing the only member from a map leaves # behind an empty map. def test_update_expression_nested_remove_singleton_map(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': {'b': 1}}) test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a.b') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == {} # DynamoDB allows an empty list. So removing the only member from a list leaves # behind an empty list. def test_update_expression_nested_remove_singleton_list(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': [1]}) test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a[0]') assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item']['a'] == [] # The DynamoDB documentation specifies: "When you use SET to update a list # element, the contents of that element are replaced with the new data that # you specify. If the element does not already exist, SET will append the # new element at the end of the list." # So if we take a three-element list a[7], and set a[7], the new element # will be put at the end of the list, not position 7 specifically. def test_nested_attribute_update_array_out_of_bounds(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': ['one', 'two', 'three']}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a[7] = :val1', ExpressionAttributeValues={':val1': 'hello'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': ['one', 'two', 'three', 'hello']} # The DynamoDB documentation also says: "If you add multiple elements # in a single SET operation, the elements are sorted in order by element # number. test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a[84] = :val1, a[37] = :val2, a[17] = :val3, a[50] = :val4', ExpressionAttributeValues={':val1': 'a1', ':val2': 'a2', ':val3': 'a3', ':val4': 'a4'}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': ['one', 'two', 'three', 'hello', 'a3', 'a2', 'a4', 'a1']} # Test what happens if we try to write to a.b, which would only make sense if # a were a nested document, but a doesn't exist, or exists and is NOT a nested # document but rather a scalar or list or something. # DynamoDB actually detects this case and prints an error: # ClientError: An error occurred (ValidationException) when calling the # UpdateItem operation: The document path provided in the update expression # is invalid for update def test_nested_attribute_update_bad_path_dot(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': 'hello', 'b': ['hi']}) with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a.c = :val1', ExpressionAttributeValues={':val1': 7}) with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b.c = :val1', ExpressionAttributeValues={':val1': 7}) with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET c.c = :val1', ExpressionAttributeValues={':val1': 7}) # Same errors for "remove" operation. with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a.c') with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE b.c') with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE c.c') # Same error when the item doesn't exist at all: p = random_string() with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET c.c = :val1', ExpressionAttributeValues={':val1': 7}) # HOWEVER, although it *is* allowed to remove a random path from a non- # existent item. I don't know why. See the next test - # test_nested_attribute_remove_from_missing_item # Though in the above test (test_nested_attribute_update_bad_path_dot) we # showed that DynamoDB does not allow REMOVE x.y if attribute x doesn't # exist - and generates a ValidationException, it turns out that if the # entire item doesn't exist, then a REMOVE x.y is silently ignored. # I don't understand why they did this. @pytest.mark.xfail(reason="for unknown reason, DynamoDB allows REMOVE x.y when item doesn't exist") def test_nested_attribute_remove_from_missing_item(test_table_s): p = random_string() test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE x.y') test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE x[0]') # Though in an above test (test_nested_attribute_update_bad_path_dot) we # showed that DynamoDB does not allow REMOVE x.y if attribute x doesn't # exist - and generates a ValidationException, if x *does* exist but y # doesn't, it's fine and the removal should just be silently ignored. def test_nested_attribute_remove_missing_leaf(test_table_s): p = random_string() item = {'p': p, 'a': {'x': 3}, 'b': ['hi']} test_table_s.put_item(Item=item) test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a.y') test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE b[7]') test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE c') # The above UpdateItem calls didn't change anything... assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == item # Similarly for other types of bad paths - using [0] on something which # doesn't exist or isn't an array. def test_nested_attribute_update_bad_path_array(test_table_s): p = random_string() test_table_s.put_item(Item={'p': p, 'a': 'hello'}) with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a[0] = :val1', ExpressionAttributeValues={':val1': 7}) with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b[0] = :val1', ExpressionAttributeValues={':val1': 7}) # Same errors for "remove" operation. with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE a[0]') with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='REMOVE b[0]') # Same error when the item doesn't exist at all: p = random_string() with pytest.raises(ClientError, match='ValidationException.*path'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b[0] = :val1', ExpressionAttributeValues={':val1': 7}) # HOWEVER, although it *is* allowed to remove a random path from a non- # existent item. I don't know why... See test_nested_attribute_remove_from_missing_item # DynamoDB Does not allow empty sets. # Trying to ask UpdateItem to put one of these in an attribute should be # forbidden. Empty lists and maps *are* allowed. # Note that in test_item.py::test_update_item_empty_attribute we checked # this with the AttributeUpdates syntax. Here we check the same with the # UpdateExpression syntax. def test_update_expression_empty_attribute(test_table_s): p = random_string() # Empty sets are *not* allowed with pytest.raises(ClientError, match='ValidationException.*empty'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = :v', ExpressionAttributeValues={':v': set()}) assert not 'Item' in test_table_s.get_item(Key={'p': p}, ConsistentRead=True) # But empty lists, maps, strings and binary blobs *are* allowed: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET d = :v1, e = :v2, f = :v3, g = :v4', ExpressionAttributeValues={':v1': [], ':v2': {}, ':v3': '', ':v4': b''}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'd': [], 'e': {}, 'f': '', 'g': b''} # The DynamoDB documentation for UpdateExpression says about "ADD" that: # "In general, we recommend using SET rather than ADD". However, it's worth # noting that unlike ADD which can treat an attribute or an item that doesn't # yet exist as zero (we tested this in test_update_expression_add_numbers_new) # SET can't do this and must be combined together with is_not_exists() to do # what ADD does. In this test we'll check that the SET alternative works as # it does in DynamoDB. def test_update_expression_set_implement_add(test_table_s): p = random_string() # "SET a = a + :val" works when a is already set test_table_s.put_item(Item={'p': p, 'a': 42}) test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = a + :val', ExpressionAttributeValues={':val': 7}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 49} # But "SET b = b + :val" doesn't work (resulting in ValidationException) # if an attribute "b" is not set in the item. It also doesn't work if # the whole item doesn't exist. # We tested something similar in test_update_expression_set_copy. with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = b + :val', ExpressionAttributeValues={':val': 7}) p1 = random_string() with pytest.raises(ClientError, match='ValidationException'): test_table_s.update_item(Key={'p': p1}, UpdateExpression='SET b = b + :val', ExpressionAttributeValues={':val': 7}) # Finally, we can use the trick "SET a = if_not_exists(a, :zero) + :val" # to allow us to treat a non-existent attribute (or even non-existent item) # as zero, and the same expression will work for both the existing and # non-existing cases, and behave like "ADD" does. # Case 1. p's a exists and its value (49) is used: test_table_s.update_item(Key={'p': p}, UpdateExpression='SET a = if_not_exists(a, :zero) + :val', ExpressionAttributeValues={':val': 3, ':zero': 0}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 52} # Case 2. p's b does not exist so zero will be used. a=52 isn't modified. test_table_s.update_item(Key={'p': p}, UpdateExpression='SET b = if_not_exists(b, :zero) + :val', ExpressionAttributeValues={':val': 5, ':zero': 0}) assert test_table_s.get_item(Key={'p': p}, ConsistentRead=True)['Item'] == {'p': p, 'a': 52, 'b': 5} # Case 3. Item p1 doesn't exist at all, so zero will be used test_table_s.update_item(Key={'p': p1}, UpdateExpression='SET a = if_not_exists(a, :zero) + :val', ExpressionAttributeValues={':val': 7, ':zero': 0}) assert test_table_s.get_item(Key={'p': p1}, ConsistentRead=True)['Item'] == {'p': p1, 'a': 7}