401dc1894c
Recently, in commit 7b30a39, we added to pytest.ini the option xfail_strict.
This option causes every time a test XPASSes, i.e., an XFAIL test actually
passes, to be considered an error and fail the test.
While this has some benefits, it's a big problem when running tests
against a reference implementation like DynamoDB or Cassandra: We
typically mark a test "xfail" if the test shows a known bug - i.e., if
the test fails on Scylla but passes on the reference system (DynamoDB
or Cassandra). This means that when running "test/cqlpy/run-cassandra"
or "test/alternator/run --aws", we expect to see many tests XPASS,
and now this will cause these runs to "fail".
So in this patch we add the xfail_strict=false to cqlpy/run-cassandra
and alternator/run --aws. This option is not added to cqlpy/run or
to alternator/run without --aws, and also doesn't affect test.py or
Jenkins.
P.S. This is another nail in the coffin of doing "cd test/alternator;
pytest --aws". You should get used to running Alternator tests through
test/alternator/run, even if you don't need to run Scylla (the "--aws"
option doesn't run Scylla).
Signed-off-by: Nadav Har'El <nyh@scylladb.com>
Closes scylladb/scylladb#28973
139 lines
5.5 KiB
Python
Executable File
139 lines
5.5 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# Use the run.py library from ../cqlpy:
|
|
import sys
|
|
sys.path.insert(1, sys.path[0] + '/../cqlpy')
|
|
import run
|
|
|
|
import os
|
|
import requests
|
|
|
|
# When tests are to be run against AWS (the "--aws" option), it is not
|
|
# necessary to start Scylla at all. All we need to do is to run pytest.
|
|
if '--aws' in sys.argv:
|
|
success = run.run_pytest(sys.path[0], ['-o', 'xfail_strict=false'] + sys.argv[1:])
|
|
exit(0 if success else 1)
|
|
|
|
# check_alternator() below uses verify=False to accept self-signed SSL
|
|
# certificates but then we get scary warnings. This trick disables them:
|
|
import urllib3
|
|
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
|
|
|
|
|
# If the first option is "--release", download that release (see
|
|
# fetch_scylla.py for supported release numbers), and use that.
|
|
# The downloaded Scylla will be cached in the directory build/<release>,
|
|
# where <release> is the specific release downloaded - E.g., if the user
|
|
# asks "--release 2022.1" and the downloaded release is 2022.1.9, it
|
|
# will be stored in build/2022.1.9.
|
|
if len(sys.argv) > 1 and sys.argv[1] == '--release':
|
|
release = sys.argv[2]
|
|
exe = run.download_precompiled_scylla(release)
|
|
run_scylla_cmd = lambda pid, dir: run.run_precompiled_scylla_cmd(exe, pid, dir)
|
|
sys.argv = sys.argv[0:1] + sys.argv[3:]
|
|
os.environ['SCYLLA'] = exe # so find_scylla() prints the right one
|
|
else:
|
|
run_scylla_cmd = run.run_scylla_cmd
|
|
|
|
print('Scylla is: ' + run.find_scylla() + '.')
|
|
|
|
extra_scylla_options = []
|
|
remove_scylla_options = []
|
|
|
|
# If the "--vnodes" option is given, drop the "tablets" experimental
|
|
# feature (turned on in run.py) so that all tests will be run with the
|
|
# old vnode-based replication instead of tablets. This option only has
|
|
# temporary usefulness, and should eventually be removed.
|
|
if '--vnodes' in sys.argv:
|
|
sys.argv.remove('--vnodes')
|
|
remove_scylla_options.append('--enable-tablets=true')
|
|
# Tablets are now enabled by default on some releases, it is not enough to remove the enable above.
|
|
extra_scylla_options.append('--enable-tablets=false')
|
|
|
|
if "-h" in sys.argv or "--help" in sys.argv:
|
|
run.run_pytest(sys.path[0], sys.argv)
|
|
exit(0)
|
|
|
|
# run_alternator_cmd runs the same as run_scylla_cmd with *additional*
|
|
# parameters, so in particular both CQL and Alternator will be enabled.
|
|
# This is useful for us because we need CQL to setup authentication.
|
|
def run_alternator_cmd(pid, dir):
|
|
(cmd, env) = run_scylla_cmd(pid, dir)
|
|
ip = run.pid_to_ip(pid)
|
|
cmd += [
|
|
'--alternator-address', ip,
|
|
'--alternator-enforce-authorization', '1',
|
|
'--alternator-write-isolation', 'only_rmw_uses_lwt',
|
|
'--alternator-streams-time-window-s', '0',
|
|
'--alternator-timeout-in-ms', '30000',
|
|
# no longer needed here because it's defined in test/cqlpy/run.py
|
|
# now that this parameter is used also by CQL's per-row TTL.
|
|
#'--alternator-ttl-period-in-seconds', '0.5',
|
|
'--alternator-allow-system-table-write=1',
|
|
# Allow testing experimental features. Following issue #9467, we need
|
|
# to add here specific experimental features as they are introduced.
|
|
# We only list here Alternator-specific experimental features - CQL
|
|
# ones are listed in test/cqlpy/run.py.
|
|
'--experimental-features=alternator-streams',
|
|
]
|
|
if '--https' in sys.argv:
|
|
run.setup_ssl_certificate(dir)
|
|
cmd += ['--alternator-https-port', '8043',
|
|
'--alternator-encryption-options', f'keyfile={dir}/scylla.key',
|
|
'--alternator-encryption-options', f'certificate={dir}/scylla.crt',
|
|
]
|
|
else:
|
|
cmd += ['--alternator-port', '8000']
|
|
cmd += extra_scylla_options
|
|
|
|
for i in remove_scylla_options:
|
|
cmd.remove(i)
|
|
|
|
# Unfortunately, earlier Scylla versions required different command line
|
|
# options to run, so for some old versions we need to drop some of the
|
|
# command line options added above, or add more options. In cqlpy/run.py's
|
|
# run_precompiled_scylla_cmd we already do this for most Scylla options,
|
|
# but here we just need to take care of the Alternator-specific options
|
|
# that we added here and were different in older versions.
|
|
if 'release' in globals():
|
|
version = release.split('~')[0].split('.')
|
|
major = [int(version[0]), int(version[1])]
|
|
print(major)
|
|
if major < [2025,4]:
|
|
cmd.remove('--alternator-allow-system-table-write=1')
|
|
|
|
return (cmd, env)
|
|
|
|
pid = run.run_with_temporary_dir(run_alternator_cmd)
|
|
ip = run.pid_to_ip(pid)
|
|
|
|
if '--https' in sys.argv:
|
|
alternator_url=f"https://{ip}:8043"
|
|
else:
|
|
alternator_url=f"http://{ip}:8000"
|
|
|
|
# Wait for both CQL and Alternator APIs to become responsive. We obviously
|
|
# need the Alternator API to test Alternator, but currently we also need
|
|
# CQL for setting up authentication.
|
|
def check_alternator(url):
|
|
try:
|
|
requests.get(url, verify=False)
|
|
except requests.ConnectionError:
|
|
raise run.NotYetUp
|
|
# Any other exception may indicate a problem, and is passed to the caller.
|
|
|
|
run.wait_for_services(pid, [
|
|
lambda: run.check_cql(ip),
|
|
lambda: check_alternator(alternator_url),
|
|
])
|
|
|
|
# Finally run pytest:
|
|
success = run.run_pytest(sys.path[0], ['--url', alternator_url] + sys.argv[1:])
|
|
|
|
run.summary = 'Alternator tests pass' if success else 'Alternator tests failure'
|
|
|
|
exit(0 if success else 1)
|
|
|
|
# Note that the run.cleanup_all() function runs now, just like on any exit
|
|
# for any reason in this script. It will delete the temporary files and
|
|
# announce the failure or success of the test (printing run.summary).
|