From 1aec4baa51aa4bce106d787a4628cfd3d5a73833 Mon Sep 17 00:00:00 2001 From: Nadav Har'El Date: Sun, 12 Apr 2020 16:01:15 +0300 Subject: [PATCH 1/6] alternator-test: add "--url" option to choose Alternator's URL The "--aws" and "--local" test options chooses between two useful default URLs - Amazon's, or http://localhost:8000 for a local installation. However, sometimes one wants to run Scylla on a different IP address or port, so in this patch we add a "--url" option to choose a specific URL to connect to. For example, "--url http://127.1.2.3:1234". We will later use this option in the alternator-test/run script, to pick a random IP address on which to run Scylla, and then run the test against this address. Signed-off-by: Nadav Har'El --- alternator-test/conftest.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/alternator-test/conftest.py b/alternator-test/conftest.py index 6218afdb27..095f8c8c0d 100644 --- a/alternator-test/conftest.py +++ b/alternator-test/conftest.py @@ -54,6 +54,8 @@ def pytest_addoption(parser): parser.addoption("--https", action="store_true", help="communicate via HTTPS protocol on port 8043 instead of HTTP when" " running against a local Scylla installation") + parser.addoption("--url", action="store", + help="communicate with given URL instead of defaults") # "dynamodb" fixture: set up client object for communicating with the DynamoDB # API. Currently this chooses either Amazon's DynamoDB in the default region @@ -70,7 +72,10 @@ def dynamodb(request): # requires us to specify dummy region and credential parameters, # otherwise the user is forced to properly configure ~/.aws even # for local runs. - local_url = 'https://localhost:8043' if request.config.getoption('https') else 'http://localhost:8000' + if request.config.getoption('url') != None: + local_url = request.config.getoption('url') + else: + local_url = 'https://localhost:8043' if request.config.getoption('https') else 'http://localhost:8000' # Disable verifying in order to be able to use self-signed TLS certificates verify = not request.config.getoption('https') return boto3.resource('dynamodb', endpoint_url=local_url, verify=verify, From 24fcc0c0ff8180e78deba7bc41604df588949875 Mon Sep 17 00:00:00 2001 From: Nadav Har'El Date: Sun, 12 Apr 2020 16:06:39 +0300 Subject: [PATCH 2/6] alternator-test: change "run" script to pick random IP address Before this patch, the Alternator tests "run" script ran Scylla on a fixed listening address, 127.0.0.1. There is a problem that there might be other concurrent runs of Scylla using the same IP address - e.g., CCM (used by dtest) uses exactly this IP address for its first node. Luckily, Linux's loopback device actually allows us to pick any of over a million addresses in 127.0.0.0/8 to listen on - we don't need to use 127.0.0.1 specifically. So the code in this patch picks an address in 127.1.*.*, so it cannot collide with CCM (which uses 127.0.0.* for up to 255 nodes). Moreover, the last two bytes of the listen address are picked based on the process ID of the run script; This allows multiple copies of this script to run concurrently - in case anybody wishes to do that. Signed-off-by: Nadav Har'El --- alternator-test/run | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/alternator-test/run b/alternator-test/run index 6b3aacbc2a..a047d16eab 100755 --- a/alternator-test/run +++ b/alternator-test/run @@ -8,7 +8,6 @@ script_path=$(dirname $(readlink -e $0)) # By default, we take the latest build/*/scylla as the executable: SCYLLA=${SCYLLA-$(ls -t "$script_path/../build/"*"/scylla" | head -1)} SCYLLA=$(readlink -f "$SCYLLA") -SCYLLA_IP=${IP-127.0.0.1} CPUSET=${CPUSET-0} # Below, we need to use python3 and the Cassandra drive to set up the @@ -21,6 +20,14 @@ then exit 1 fi +# Pick a loopback IP address for Scylla to run, in an attempt not to collide +# other concurrent runs of Scylla. CCM uses 127.0.0., so if we use +# 127.1.*.* which cannot collide with it. Moreover, we'll take the last two +# bytes of the address from the current process - so as to allow multiple +# concurrent runs of this code to use a different address. +SCYLLA_IP=127.1.$(($$ >> 8 & 255)).$(($$ & 255)) +echo "Running Scylla on $SCYLLA_IP" + tmp_dir=/tmp/alternator-test-$$ mkdir $tmp_dir @@ -51,6 +58,7 @@ trap 'cleanup' EXIT # to work. We only need to do this if the "--https" option was explicitly # passed - otherwise the test would not use HTTPS anyway. alternator_port_option="--alternator-port=8000" +alternator_url="http://$SCYLLA_IP:8000" for i do if [ "$i" = --https ] @@ -58,17 +66,20 @@ do openssl genrsa 2048 > "$tmp_dir/scylla.key" openssl req -new -x509 -nodes -sha256 -days 365 -subj "/C=IL/ST=None/L=None/O=None/OU=None/CN=example.com" -key "$tmp_dir/scylla.key" -out "$tmp_dir/scylla.crt" alternator_port_option="--alternator-https-port=8043" + alternator_url="https://$SCYLLA_IP:8043" fi done -"$SCYLLA" --options-file "$script_path/../conf/scylla.yaml" \ - --alternator-address $SCYLLA_IP \ +"$SCYLLA" --options-file "$source_path/conf/scylla.yaml" \ + --alternator-address $SCYLLA_IP \ $alternator_port_option \ --alternator-enforce-authorization=1 \ --developer-mode=1 \ --ring-delay-ms 0 --collectd 0 \ --cpuset "$CPUSET" -m 1G \ - --api-address $SCYLLA_IP --rpc-address $SCYLLA_IP \ + --api-address $SCYLLA_IP \ + --rpc-address $SCYLLA_IP \ --listen-address $SCYLLA_IP \ + --prometheus-address $SCYLLA_IP \ --seed-provider-parameters seeds=$SCYLLA_IP \ --workdir "$tmp_dir" \ --server-encryption-options keyfile="$tmp_dir/scylla.key" \ @@ -81,7 +92,7 @@ SCYLLA_PROCESS=$! # test. This requires connecting to Scylla with CQL - we'll wait up for # one minute for this to work: setup_authentication() { - python3 -c 'from cassandra.cluster import Cluster; Cluster().connect().execute("INSERT INTO system_auth.roles (role, salted_hash) VALUES ('\''alternator'\'', '\''secret_pass'\'')")' + python3 -c 'from cassandra.cluster import Cluster; Cluster(["'$SCYLLA_IP'"]).connect().execute("INSERT INTO system_auth.roles (role, salted_hash) VALUES ('\''alternator'\'', '\''secret_pass'\'')")' } echo "Scylla is: $SCYLLA." echo -n "Booting Scylla..." @@ -127,7 +138,8 @@ else fi cd "$script_path" -pytest "$@" +set +e +pytest --url $alternator_url "$@" code=$? case $code in 0) summary="Alternator tests pass";; From 36e44972f102d5375970e84fc5fa795931852d7e Mon Sep 17 00:00:00 2001 From: Nadav Har'El Date: Thu, 26 Mar 2020 23:18:41 +0200 Subject: [PATCH 3/6] test.py: flag for aborting tests with SIGTERM, not SIGKILL Today, if test.py is interrupted with SIGINT or SIGTERM, the ongoing test is killed with SIGKILL. Some types of tests - such as Alternator's test - may depend on being killed politely (e.g., with SIGTERM) to clean up files. We cannot yet change the signal to SIGTERM for all tests, because Seastar tests often don't deal well with signals, but we can at least add a flag that certain test types - that know they can be killed gently - will use. Signed-off-by: Nadav Har'El --- test.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/test.py b/test.py index 4cd7fedcb7..d88785aae1 100755 --- a/test.py +++ b/test.py @@ -375,7 +375,7 @@ class TabularConsoleOutput: print(msg) -async def run_test(test, options): +async def run_test(test, options, gentle_kill=False): """Run test program, return True if success else False""" with open(test.log_filename, "wb") as log: @@ -423,7 +423,10 @@ async def run_test(test, options): return True except (asyncio.TimeoutError, asyncio.CancelledError) as e: if process is not None: - process.kill() + if gentle_kill: + process.terminate() + else: + process.kill() stdout, _ = await process.communicate() if isinstance(e, asyncio.TimeoutError): report_error("Test timed out") From 0ae31369002c8ff9ef95ac04868fcb6de3fcc805 Mon Sep 17 00:00:00 2001 From: Nadav Har'El Date: Thu, 26 Mar 2020 23:27:00 +0200 Subject: [PATCH 4/6] test.py: add new test type "Run" This patch adds a new test type, "Run". A test subdirectory of type "Run" has a script called "run" which is expected to run all the tests in that directory. This will be used, in the next patch, by the Alternator functional tests. These tests indeed have a "run" script, which runs Scylla and then runs *all* of Alternator's tests, finishing fairly quickly (in less than a minute). All of that will become one test.py test. Signed-off-by: Nadav Har'El --- test.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/test.py b/test.py index d88785aae1..568f03cd96 100755 --- a/test.py +++ b/test.py @@ -203,6 +203,17 @@ class CqlTestSuite(TestSuite): def pattern(self): return "*_test.cql" +class RunTestSuite(TestSuite): + """TestSuite for test directory with a 'run' script """ + + def add_test(self, shortname, mode, options): + test = RunTest(self.next_id, shortname, self, mode, options) + self.tests.append(test) + + @property + def pattern(self): + return "run" + class Test: """Base class for CQL, Unit and Boost tests""" @@ -332,6 +343,23 @@ class CqlTest(Test): if self.is_equal_result is False: print_unidiff(self.result, self.reject) +class RunTest(Test): + """Run tests in a directory started by a run script""" + + def __init__(self, test_no, shortname, suite, mode, options): + super().__init__(test_no, shortname, suite, mode, options) + self.path = os.path.join(suite.path, shortname) + self.args = "" + + def print_summary(self): + print("Output of {} {}:".format(self.path, " ".join(self.args))) + print(read_log(self.log_filename)) + + async def run(self, options): + # This test can and should be killed gently, with SIGTERM, not with SIGKILL + self.success = await run_test(self, options, gentle_kill=True) + logging.info("Test #%d %s", self.id, "succeeded" if self.success else "failed ") + return self class TabularConsoleOutput: """Print test progress to the console""" From 0cccb5a630d9adcbfcb47df85b26a4abe529c98a Mon Sep 17 00:00:00 2001 From: Nadav Har'El Date: Thu, 2 Apr 2020 18:52:28 +0300 Subject: [PATCH 5/6] test.py: add xunit XML output file for "Run" tests Assumes that "Run" tests can take the --junit-xml= option, and pass it to ask the test to generate an XML summary of the run to a file like testlog/dev/xml/run.1.xunit.xml. This option is honored by the Alternator tests. Signed-off-by: Nadav Har'El --- test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test.py b/test.py index 568f03cd96..9d14dbfe54 100755 --- a/test.py +++ b/test.py @@ -349,7 +349,8 @@ class RunTest(Test): def __init__(self, test_no, shortname, suite, mode, options): super().__init__(test_no, shortname, suite, mode, options) self.path = os.path.join(suite.path, shortname) - self.args = "" + self.xmlout = os.path.join(options.tmpdir, self.mode, "xml", self.uname + ".xunit.xml") + self.args = ["--junit-xml={}".format(self.xmlout)] def print_summary(self): print("Output of {} {}:".format(self.path, " ".join(self.args))) From 4e2bf28b8455527237baabe629f01c775fe4e48c Mon Sep 17 00:00:00 2001 From: Nadav Har'El Date: Thu, 26 Mar 2020 23:43:13 +0200 Subject: [PATCH 6/6] alternator-test: make Alternator tests runnable from test.py To make the tests in alternator-test runnable by test.py, we need to move the directory alternator-test/ to test/alternator, because test.py only looks for tests in subdirectories of test/. Then, we need to create a test/alternator/suite.yaml saying that this test directory is of type "Run", i.e., it has a single run script "run" which runs all its tests. The "run" script had to be slightly modified to be aware of its new location relative to the source directory. To run the Alternator tests from test.py, do: ./test.py --mode dev alternator Note that in this version, the "--mode" has no effect - test/alternator/run always runs the latest compiled Scylla, regardless of the chosen mode. The Alternator tests can still be run manually and individually against a running Scylla or DynamoDB as before - just go to the test/alternator directory (instead of alternator-test previously) and run "pytest" with the desired parameters. Fixes #6046 Signed-off-by: Nadav Har'El --- {alternator-test => test/alternator}/README.md | 0 {alternator-test => test/alternator}/conftest.py | 0 {alternator-test => test/alternator}/run | 3 ++- test/alternator/suite.yaml | 1 + {alternator-test => test/alternator}/test_authorization.py | 0 {alternator-test => test/alternator}/test_batch.py | 0 .../alternator}/test_condition_expression.py | 0 .../alternator}/test_describe_endpoints.py | 0 {alternator-test => test/alternator}/test_describe_table.py | 0 {alternator-test => test/alternator}/test_expected.py | 0 {alternator-test => test/alternator}/test_gsi.py | 0 {alternator-test => test/alternator}/test_health.py | 0 {alternator-test => test/alternator}/test_item.py | 0 .../alternator}/test_key_condition_expression.py | 0 {alternator-test => test/alternator}/test_lsi.py | 0 {alternator-test => test/alternator}/test_manual_requests.py | 0 {alternator-test => test/alternator}/test_nested.py | 0 .../alternator}/test_projection_expression.py | 0 {alternator-test => test/alternator}/test_query.py | 0 {alternator-test => test/alternator}/test_returnvalues.py | 0 {alternator-test => test/alternator}/test_scan.py | 0 {alternator-test => test/alternator}/test_scylla.py | 0 {alternator-test => test/alternator}/test_table.py | 0 {alternator-test => test/alternator}/test_tag.py | 0 {alternator-test => test/alternator}/test_update_expression.py | 0 {alternator-test => test/alternator}/util.py | 0 26 files changed, 3 insertions(+), 1 deletion(-) rename {alternator-test => test/alternator}/README.md (100%) rename {alternator-test => test/alternator}/conftest.py (100%) rename {alternator-test => test/alternator}/run (98%) create mode 100644 test/alternator/suite.yaml rename {alternator-test => test/alternator}/test_authorization.py (100%) rename {alternator-test => test/alternator}/test_batch.py (100%) rename {alternator-test => test/alternator}/test_condition_expression.py (100%) rename {alternator-test => test/alternator}/test_describe_endpoints.py (100%) rename {alternator-test => test/alternator}/test_describe_table.py (100%) rename {alternator-test => test/alternator}/test_expected.py (100%) rename {alternator-test => test/alternator}/test_gsi.py (100%) rename {alternator-test => test/alternator}/test_health.py (100%) rename {alternator-test => test/alternator}/test_item.py (100%) rename {alternator-test => test/alternator}/test_key_condition_expression.py (100%) rename {alternator-test => test/alternator}/test_lsi.py (100%) rename {alternator-test => test/alternator}/test_manual_requests.py (100%) rename {alternator-test => test/alternator}/test_nested.py (100%) rename {alternator-test => test/alternator}/test_projection_expression.py (100%) rename {alternator-test => test/alternator}/test_query.py (100%) rename {alternator-test => test/alternator}/test_returnvalues.py (100%) rename {alternator-test => test/alternator}/test_scan.py (100%) rename {alternator-test => test/alternator}/test_scylla.py (100%) rename {alternator-test => test/alternator}/test_table.py (100%) rename {alternator-test => test/alternator}/test_tag.py (100%) rename {alternator-test => test/alternator}/test_update_expression.py (100%) rename {alternator-test => test/alternator}/util.py (100%) diff --git a/alternator-test/README.md b/test/alternator/README.md similarity index 100% rename from alternator-test/README.md rename to test/alternator/README.md diff --git a/alternator-test/conftest.py b/test/alternator/conftest.py similarity index 100% rename from alternator-test/conftest.py rename to test/alternator/conftest.py diff --git a/alternator-test/run b/test/alternator/run similarity index 98% rename from alternator-test/run rename to test/alternator/run index a047d16eab..1cf04c70dd 100755 --- a/alternator-test/run +++ b/test/alternator/run @@ -4,9 +4,10 @@ set -e script_path=$(dirname $(readlink -e $0)) +source_path=$script_path/../.. # By default, we take the latest build/*/scylla as the executable: -SCYLLA=${SCYLLA-$(ls -t "$script_path/../build/"*"/scylla" | head -1)} +SCYLLA=${SCYLLA-$(ls -t "$source_path/build/"*"/scylla" | head -1)} SCYLLA=$(readlink -f "$SCYLLA") CPUSET=${CPUSET-0} diff --git a/test/alternator/suite.yaml b/test/alternator/suite.yaml new file mode 100644 index 0000000000..8919604e66 --- /dev/null +++ b/test/alternator/suite.yaml @@ -0,0 +1 @@ +type: Run diff --git a/alternator-test/test_authorization.py b/test/alternator/test_authorization.py similarity index 100% rename from alternator-test/test_authorization.py rename to test/alternator/test_authorization.py diff --git a/alternator-test/test_batch.py b/test/alternator/test_batch.py similarity index 100% rename from alternator-test/test_batch.py rename to test/alternator/test_batch.py diff --git a/alternator-test/test_condition_expression.py b/test/alternator/test_condition_expression.py similarity index 100% rename from alternator-test/test_condition_expression.py rename to test/alternator/test_condition_expression.py diff --git a/alternator-test/test_describe_endpoints.py b/test/alternator/test_describe_endpoints.py similarity index 100% rename from alternator-test/test_describe_endpoints.py rename to test/alternator/test_describe_endpoints.py diff --git a/alternator-test/test_describe_table.py b/test/alternator/test_describe_table.py similarity index 100% rename from alternator-test/test_describe_table.py rename to test/alternator/test_describe_table.py diff --git a/alternator-test/test_expected.py b/test/alternator/test_expected.py similarity index 100% rename from alternator-test/test_expected.py rename to test/alternator/test_expected.py diff --git a/alternator-test/test_gsi.py b/test/alternator/test_gsi.py similarity index 100% rename from alternator-test/test_gsi.py rename to test/alternator/test_gsi.py diff --git a/alternator-test/test_health.py b/test/alternator/test_health.py similarity index 100% rename from alternator-test/test_health.py rename to test/alternator/test_health.py diff --git a/alternator-test/test_item.py b/test/alternator/test_item.py similarity index 100% rename from alternator-test/test_item.py rename to test/alternator/test_item.py diff --git a/alternator-test/test_key_condition_expression.py b/test/alternator/test_key_condition_expression.py similarity index 100% rename from alternator-test/test_key_condition_expression.py rename to test/alternator/test_key_condition_expression.py diff --git a/alternator-test/test_lsi.py b/test/alternator/test_lsi.py similarity index 100% rename from alternator-test/test_lsi.py rename to test/alternator/test_lsi.py diff --git a/alternator-test/test_manual_requests.py b/test/alternator/test_manual_requests.py similarity index 100% rename from alternator-test/test_manual_requests.py rename to test/alternator/test_manual_requests.py diff --git a/alternator-test/test_nested.py b/test/alternator/test_nested.py similarity index 100% rename from alternator-test/test_nested.py rename to test/alternator/test_nested.py diff --git a/alternator-test/test_projection_expression.py b/test/alternator/test_projection_expression.py similarity index 100% rename from alternator-test/test_projection_expression.py rename to test/alternator/test_projection_expression.py diff --git a/alternator-test/test_query.py b/test/alternator/test_query.py similarity index 100% rename from alternator-test/test_query.py rename to test/alternator/test_query.py diff --git a/alternator-test/test_returnvalues.py b/test/alternator/test_returnvalues.py similarity index 100% rename from alternator-test/test_returnvalues.py rename to test/alternator/test_returnvalues.py diff --git a/alternator-test/test_scan.py b/test/alternator/test_scan.py similarity index 100% rename from alternator-test/test_scan.py rename to test/alternator/test_scan.py diff --git a/alternator-test/test_scylla.py b/test/alternator/test_scylla.py similarity index 100% rename from alternator-test/test_scylla.py rename to test/alternator/test_scylla.py diff --git a/alternator-test/test_table.py b/test/alternator/test_table.py similarity index 100% rename from alternator-test/test_table.py rename to test/alternator/test_table.py diff --git a/alternator-test/test_tag.py b/test/alternator/test_tag.py similarity index 100% rename from alternator-test/test_tag.py rename to test/alternator/test_tag.py diff --git a/alternator-test/test_update_expression.py b/test/alternator/test_update_expression.py similarity index 100% rename from alternator-test/test_update_expression.py rename to test/alternator/test_update_expression.py diff --git a/alternator-test/util.py b/test/alternator/util.py similarity index 100% rename from alternator-test/util.py rename to test/alternator/util.py