test/alternator: tests for schema of Alternator table

This patch introduces a new test that exposed a previously unknown bug,
Refs #26914:

Recently we saw a lot of patches that change how we create new schemas
(keyspaces and tables), sometimes changing various long-time defaults.
We started to worry that perhaps some of these defaults were applied only
to CQL and not to Alternator. For example, in Refs #26307 we wondered if
perhaps the default "speculative_retry" option is different in Alternator
than in CQL.

This patch includes a new test file test/alternator/test_cql_schema.py,
with tests for verifying how Alternator configures the underlying tables
it creates. This test shows that the "speculative_retry" doesn't have
this suspected bug - it defaults to "99.0PERCENTILE" in both CQL and
Alternator. But unfortunately, we do have this bug with the "compression"
option:

It turns out that recently (commit adf9c42, Refs #26610) we changed the
default sstable compressor from LZ4Compressor to LZ4WithDictsCompressor,
but the change was only applied to CQL, not Alternator. So the test that
"compression" is the same in both fails - and marked "xfails" and
I created a new issue to track it - #26914.

Another test verifies that Alternators "auxiliary" tables - holding
GSIs, LSIs and Streams - have the same default properties as the base
table. This currently seems to hold (there is no bug).

Signed-off-by: Nadav Har'El <nyh@scylladb.com>

test/alternator/test_cql_schema.py

@@ -0,0 +1,171 @@
+# Copyright 2025-present ScyllaDB
+#
+# SPDX-License-Identifier: LicenseRef-ScyllaDB-Source-Available-1.0
+
+# Tests for the underlying CQL schema (keyspace and table) which is used to
+# store an Alternator table and all its sub-tables (holding GSIs, LSIs and
+# CDC logs).
+# Arguably, since those properties aren't directly visible through the
+# DynamoDB API, we do not really need to guarantee backward compatibility
+# in this area, so we shouldn't have tests for it. However, these tests are
+# still useful for understanding what the underlying CQL schema looks like,
+# and to prevent unintended drift between CQL's and Alternator's defaults
+# or unintended drift to our on-disk encoding (see issue #19770) our
+# existing storage. If we ever do *intended* changes to these properties
+# of Alternator tables, we will need to change these tests.
+#
+# Because this file is all about testing the Scylla-only CQL-based APIs,
+# all tests in this file are skipped when running against Amazon DynamoDB.
+#
+# There are additional files with tests related to configuring certain
+# properties of Alternator tables through CQL: See test_cql_rbac.py for
+# configuring role-based access control, and test_service_levels.py for
+# configuring service levels - both need to be done through CQL.
+
+import string
+import pytest
+
+from .util import full_query, global_random, new_test_table
+from .test_scylla import this_dc
+
+# All tests in this file are scylla-only
+@pytest.fixture(scope="function", autouse=True)
+def all_tests_are_scylla_only(scylla_only):
+    pass
+
+# Utility function for getting (using Alternator's system-table access)
+# the options set for the given CQL table given by a keyspace, name and
+# is_view triplet. The is_view boolean decides if the name is assumed to
+# refer to a materialized view or a base table. In Scylla's system tables
+# we have a separate system table listing tables and views.
+def scylla_get_schema(dynamodb, ks_name_and_isview):
+    ks, name, is_view = ks_name_and_isview
+    what = 'view' if is_view else 'table'
+    sys = dynamodb.Table(f'.scylla.alternator.system_schema.{what}s')
+    # We need to read just a single key, but strangely Alternator doesn't
+    # allow GetItem on system tables, just Query/Scan, so we use Query:
+    res = full_query(sys,
+        KeyConditionExpression=f'keyspace_name=:ks and {what}_name=:name',
+        ExpressionAttributeValues={':ks': ks, ':name' : name})
+    assert len(res) == 1
+    return res[0]
+
+# Utility functions for getting the keyspace and table name used to store
+# an Alternator base-table, GSI, LSI and Streams respectively. Tests below
+# will use those functions, so will ensure that this table naming doesn't
+# accidentally change. These functions return a triplet ks,name,is_view
+# as expected by scylla_get_schema().
+def cql_table_for(table):
+    return 'alternator_'+table.name, table.name, False
+
+def cql_table_for_gsi(table, gsi):
+    return 'alternator_'+table.name, table.name + ':' + gsi, True
+
+def cql_table_for_lsi(table, lsi):
+    return 'alternator_'+table.name, table.name + '!:' + lsi, True
+
+def cql_table_for_cdclog(table):
+    return 'alternator_'+table.name, table.name + '_scylla_cdc_log', False
+
+# Confirm that for Alternator table table_name, a CQL table is created with
+# keyspace "alternator_{table_name}" and table "{table_name}".
+def test_cql_keyspace_and_table(dynamodb, test_table):
+    scylla_get_schema(dynamodb, cql_table_for(test_table))
+
+# The fixtures cql_keyspace, cql_table create a new CQL keyspace and table,
+# so we can compare their properties to those of a table created by Alternator.
+def unique_name():
+    return ''.join(global_random.choice(string.ascii_lowercase) for x in range(20))
+
+@pytest.fixture(scope='module')
+def cql_keyspace(cql, this_dc):
+    name = unique_name()
+    cql.execute("CREATE KEYSPACE " + name + " WITH REPLICATION = { 'class' : 'NetworkTopologyStrategy', '" + this_dc + "' : 1 }")
+    yield name
+    cql.execute("DROP KEYSPACE " + name)
+
+@pytest.fixture(scope='module')
+def cql_table(cql, cql_keyspace):
+    cf = unique_name()
+    cql.execute(f'CREATE TABLE {cql_keyspace}.{cf} (k INT PRIMARY KEY)')
+    yield cf
+    cql.execute(f'DROP TABLE {cql_keyspace}.{cf}')
+
+# Check that a new Alternator table gets the same default configuration as
+# a new CQL table - for a selection of configuration parameters like
+# compression and speculative_retry. We don't check what it is (e.g., at the
+# time of this writing, compression is LZ4 compression), but we want it to
+# have the same default in Alternator as in CQL.
+# Reproduces #26914 ('compression' difference).
+@pytest.mark.parametrize('option',
+    [pytest.param('compression', marks=pytest.mark.xfail(reason='#issue #26914')),
+    'speculative_retry'])
+def test_alternator_vs_cql(dynamodb, test_table, cql_keyspace, cql_table, option):
+    alternator_schema = scylla_get_schema(dynamodb, cql_table_for(test_table))
+    cql_schema = scylla_get_schema(dynamodb, (cql_keyspace, cql_table, False))
+    assert alternator_schema[option] == cql_schema[option]
+    # If you're curious what are the current default values of these options,
+    # run this test with "-s" to see the print output.
+    print(f'{option}: {alternator_schema[option]}')
+
+# Fixture for an Alternator table with a GSI, LSI and Streams, allowing us
+# to check the schemas of not just the Alternator table itself, but also
+# its materialized views and CDC logs.
+@pytest.fixture(scope='module')
+def table1(dynamodb):
+    with new_test_table(dynamodb,
+        # Alternator Streams is expected to fail with tablets due to #23838.
+        # To ensure that this test still runs, instead of xfailing it, we
+        # temporarily coerce Alternator to avoid using default tablets
+        # setting, even if it's available. Remove this "Tags=" line when
+        # issue #23838 is solved.
+        Tags=[{'Key': 'system:initial_tablets', 'Value': 'none'}],
+        KeySchema=[
+            # Must have both hash key and range key to allow LSI creation
+            { 'AttributeName': 'p', 'KeyType': 'HASH' },
+            { 'AttributeName': 'c', 'KeyType': 'RANGE' }
+        ],
+        AttributeDefinitions=[
+            { 'AttributeName': 'p', 'AttributeType': 'S' },
+            { 'AttributeName': 'c', 'AttributeType': 'S' },
+            { 'AttributeName': 'x', 'AttributeType': 'S' },
+        ],
+        LocalSecondaryIndexes=[
+            {   'IndexName': 'lsi_name',
+                'KeySchema': [
+                    { 'AttributeName': 'p', 'KeyType': 'HASH' },
+                    { 'AttributeName': 'x', 'KeyType': 'RANGE' },
+                ],
+                'Projection': { 'ProjectionType': 'ALL' }
+            }
+        ],
+        GlobalSecondaryIndexes=[
+            {   'IndexName': 'gsi_name',
+                'KeySchema': [{ 'AttributeName': 'x', 'KeyType': 'HASH' }],
+                'Projection': { 'ProjectionType': 'ALL' }
+            }
+        ],
+        StreamSpecification={
+            'StreamEnabled': True, 'StreamViewType': 'KEYS_ONLY'
+        }
+        ) as table:
+        yield table
+
+# Check that Alternator's auxiliary tables - holding a GSI, LSI or CDC log,
+# get the same configuration an Alternator base table. In the previous test
+# (test_alternator_vs_cql) we already checked that the base-table
+# configuration is the same as CQL's default.
+@pytest.mark.parametrize('option',
+    ['compression',
+    'speculative_retry'])
+def test_alternator_aux_tables(dynamodb, table1, option):
+    alternator_base_schema = scylla_get_schema(dynamodb, cql_table_for(table1))
+    # Check GSI table:
+    alternator_gsi_schema = scylla_get_schema(dynamodb, cql_table_for_gsi(table1, 'gsi_name'))
+    assert alternator_base_schema[option] == alternator_gsi_schema[option]
+    # Check LSI table:
+    alternator_lsi_schema = scylla_get_schema(dynamodb, cql_table_for_lsi(table1, 'lsi_name'))
+    assert alternator_base_schema[option] == alternator_lsi_schema[option]
+    # Check Streams log table:
+    alternator_cdc_schema = scylla_get_schema(dynamodb, cql_table_for_cdclog(table1))
+    assert alternator_base_schema[option] == alternator_cdc_schema[option]