mirror of
https://github.com/scylladb/scylladb.git
synced 2026-04-28 04:06:59 +00:00
b556356a7d
Although different partitions are returned by a Scan in (seemingly) random order, items in a single partition need to be returned sorted by their sort key. This adds a test to verify this. This patch adds to the filled_test_table fixture, which until now had just one item in each partition, another partition (with the key "long") with 164 additional items. The test_scan_sort_order_string test then scans this table, and verifies that the items are really returned in sorted order. The sort order is, of course, string order. So we have the first item with sort key "1", then "10", then "100", then "101", "102", etc. When we implement numeric keys we'll need to add a version of this test which uses a numeric clustering key and verifies the sort order is numeric. Signed-off-by: Nadav Har'El <nyh@scylladb.com>
73 lines
2.8 KiB
Python
73 lines
2.8 KiB
Python
# Tests for the Scan operation
|
|
|
|
import random
|
|
import string
|
|
|
|
import pytest
|
|
from botocore.exceptions import ClientError
|
|
|
|
def set_of_frozen_elements(list_of_dicts):
|
|
return {frozenset(item.items()) for item in list_of_dicts}
|
|
|
|
# Test that scanning works fine with/without pagination
|
|
def test_scan_basic(filled_test_table):
|
|
test_table, items = filled_test_table
|
|
for limit in [None,1,2,4,33,50,100,9007,16*1024*1024]:
|
|
pos = None
|
|
got_items = []
|
|
while True:
|
|
if limit:
|
|
response = test_table.scan(Limit=limit, ExclusiveStartKey=pos) if pos else test_table.scan(Limit=limit)
|
|
assert len(response['Items']) <= limit
|
|
else:
|
|
response = test_table.scan(ExclusiveStartKey=pos) if pos else test_table.scan()
|
|
pos = response.get('LastEvaluatedKey', None)
|
|
got_items += response['Items']
|
|
if not pos:
|
|
break
|
|
|
|
assert len(items) == len(got_items)
|
|
assert set_of_frozen_elements(items) == set_of_frozen_elements(got_items)
|
|
|
|
def test_scan_with_paginator(dynamodb, filled_test_table):
|
|
test_table, items = filled_test_table
|
|
paginator = dynamodb.meta.client.get_paginator('scan')
|
|
|
|
got_items = []
|
|
for page in paginator.paginate(TableName=test_table.name):
|
|
got_items += page['Items']
|
|
|
|
assert len(items) == len(got_items)
|
|
assert set_of_frozen_elements(items) == set_of_frozen_elements(got_items)
|
|
|
|
for page_size in [1, 17, 1234]:
|
|
got_items = []
|
|
for page in paginator.paginate(TableName=test_table.name, PaginationConfig={'PageSize': page_size}):
|
|
got_items += page['Items']
|
|
|
|
assert len(items) == len(got_items)
|
|
assert set_of_frozen_elements(items) == set_of_frozen_elements(got_items)
|
|
|
|
# Although partitions are scanned in seemingly-random order, inside a
|
|
# partition items must be returned by Scan sorted in sort-key order.
|
|
# This test verifies this, for string sort key. We'll need separate
|
|
# tests for the other sort-key types (number and binary)
|
|
def test_scan_sort_order_string(filled_test_table):
|
|
test_table, items = filled_test_table
|
|
pos = None
|
|
got_items = []
|
|
while True:
|
|
response = test_table.scan(ExclusiveStartKey=pos) if pos else test_table.scan()
|
|
pos = response.get('LastEvaluatedKey', None)
|
|
got_items += response['Items']
|
|
if not pos:
|
|
break
|
|
assert len(items) == len(got_items)
|
|
# Extract just the sort key ("c") from the partition "long"
|
|
items_long = [x['c'] for x in items if x['p'] == 'long']
|
|
got_items_long = [x['c'] for x in got_items if x['p'] == 'long']
|
|
# Verify that got_items_long are already sorted (in string order)
|
|
assert sorted(got_items_long) == got_items_long
|
|
# Verify that got_items_long are a sorted version of the expected items_long
|
|
assert sorted(items_long) == got_items_long
|