diff --git a/test.py b/test.py
index 4cd7fedcb7..9d14dbfe54 100755
--- a/test.py
+++ b/test.py
@@ -203,6 +203,17 @@ class CqlTestSuite(TestSuite):
     def pattern(self):
         return "*_test.cql"
 
+class RunTestSuite(TestSuite):
+    """TestSuite for test directory with a 'run' script """
+
+    def add_test(self, shortname, mode, options):
+        test = RunTest(self.next_id, shortname, self, mode, options)
+        self.tests.append(test)
+
+    @property
+    def pattern(self):
+        return "run"
+
 
 class Test:
     """Base class for CQL, Unit and Boost tests"""
@@ -332,6 +343,24 @@ class CqlTest(Test):
         if self.is_equal_result is False:
             print_unidiff(self.result, self.reject)
 
+class RunTest(Test):
+    """Run tests in a directory started by a run script"""
+
+    def __init__(self, test_no, shortname, suite, mode, options):
+        super().__init__(test_no, shortname, suite, mode, options)
+        self.path = os.path.join(suite.path, shortname)
+        self.xmlout = os.path.join(options.tmpdir, self.mode, "xml", self.uname + ".xunit.xml")
+        self.args = ["--junit-xml={}".format(self.xmlout)]
+
+    def print_summary(self):
+        print("Output of {} {}:".format(self.path, " ".join(self.args)))
+        print(read_log(self.log_filename))
+
+    async def run(self, options):
+        # This test can and should be killed gently, with SIGTERM, not with SIGKILL
+        self.success = await run_test(self, options, gentle_kill=True)
+        logging.info("Test #%d %s", self.id, "succeeded" if self.success else "failed ")
+        return self
 
 class TabularConsoleOutput:
     """Print test progress to the console"""
@@ -375,7 +404,7 @@ class TabularConsoleOutput:
             print(msg)
 
 
-async def run_test(test, options):
+async def run_test(test, options, gentle_kill=False):
     """Run test program, return True if success else False"""
 
     with open(test.log_filename, "wb") as log:
@@ -423,7 +452,10 @@ async def run_test(test, options):
             return True
         except (asyncio.TimeoutError, asyncio.CancelledError) as e:
             if process is not None:
-                process.kill()
+                if gentle_kill:
+                    process.terminate()
+                else:
+                    process.kill()
                 stdout, _ = await process.communicate()
             if isinstance(e, asyncio.TimeoutError):
                 report_error("Test timed out")
diff --git a/alternator-test/README.md b/test/alternator/README.md
similarity index 100%
rename from alternator-test/README.md
rename to test/alternator/README.md
diff --git a/alternator-test/conftest.py b/test/alternator/conftest.py
similarity index 96%
rename from alternator-test/conftest.py
rename to test/alternator/conftest.py
index 6218afdb27..095f8c8c0d 100644
--- a/alternator-test/conftest.py
+++ b/test/alternator/conftest.py
@@ -54,6 +54,8 @@ def pytest_addoption(parser):
     parser.addoption("--https", action="store_true",
         help="communicate via HTTPS protocol on port 8043 instead of HTTP when"
             " running against a local Scylla installation")
+    parser.addoption("--url", action="store",
+        help="communicate with given URL instead of defaults")
 
 # "dynamodb" fixture: set up client object for communicating with the DynamoDB
 # API. Currently this chooses either Amazon's DynamoDB in the default region
@@ -70,7 +72,10 @@ def dynamodb(request):
         # requires us to specify dummy region and credential parameters,
         # otherwise the user is forced to properly configure ~/.aws even
         # for local runs.
-        local_url = 'https://localhost:8043' if request.config.getoption('https') else 'http://localhost:8000'
+        if request.config.getoption('url') != None:
+            local_url = request.config.getoption('url')
+        else:
+            local_url = 'https://localhost:8043' if request.config.getoption('https') else 'http://localhost:8000'
         # Disable verifying in order to be able to use self-signed TLS certificates
         verify = not request.config.getoption('https')
         return boto3.resource('dynamodb', endpoint_url=local_url, verify=verify,
diff --git a/alternator-test/run b/test/alternator/run
similarity index 79%
rename from alternator-test/run
rename to test/alternator/run
index 6b3aacbc2a..1cf04c70dd 100755
--- a/alternator-test/run
+++ b/test/alternator/run
@@ -4,11 +4,11 @@
 set -e
 
 script_path=$(dirname $(readlink -e $0))
+source_path=$script_path/../..
 
 # By default, we take the latest build/*/scylla as the executable:
-SCYLLA=${SCYLLA-$(ls -t "$script_path/../build/"*"/scylla" | head -1)}
+SCYLLA=${SCYLLA-$(ls -t "$source_path/build/"*"/scylla" | head -1)}
 SCYLLA=$(readlink -f "$SCYLLA")
-SCYLLA_IP=${IP-127.0.0.1}
 CPUSET=${CPUSET-0}
 
 # Below, we need to use python3 and the Cassandra drive to set up the
@@ -21,6 +21,14 @@ then
     exit 1
 fi
 
+# Pick a loopback IP address for Scylla to run, in an attempt not to collide
+# other concurrent runs of Scylla. CCM uses 127.0.0.<nodenum>, so if we use
+# 127.1.*.* which cannot collide with it. Moreover, we'll take the last two
+# bytes of the address from the current process - so as to allow multiple
+# concurrent runs of this code to use a different address.
+SCYLLA_IP=127.1.$(($$ >> 8 & 255)).$(($$ & 255))
+echo "Running Scylla on $SCYLLA_IP"
+
 tmp_dir=/tmp/alternator-test-$$
 mkdir $tmp_dir
 
@@ -51,6 +59,7 @@ trap 'cleanup' EXIT
 # to work. We only need to do this if the "--https" option was explicitly
 # passed - otherwise the test would not use HTTPS anyway.
 alternator_port_option="--alternator-port=8000"
+alternator_url="http://$SCYLLA_IP:8000"
 for i
 do
     if [ "$i" = --https ]
@@ -58,17 +67,20 @@ do
         openssl genrsa 2048 > "$tmp_dir/scylla.key"
         openssl req -new -x509 -nodes -sha256 -days 365 -subj "/C=IL/ST=None/L=None/O=None/OU=None/CN=example.com" -key "$tmp_dir/scylla.key" -out "$tmp_dir/scylla.crt"
         alternator_port_option="--alternator-https-port=8043"
+        alternator_url="https://$SCYLLA_IP:8043"
     fi
 done
-"$SCYLLA" --options-file "$script_path/../conf/scylla.yaml" \
-         --alternator-address $SCYLLA_IP \
+"$SCYLLA" --options-file "$source_path/conf/scylla.yaml" \
+        --alternator-address $SCYLLA_IP \
         $alternator_port_option \
         --alternator-enforce-authorization=1 \
         --developer-mode=1 \
         --ring-delay-ms 0 --collectd 0 \
         --cpuset "$CPUSET" -m 1G \
-        --api-address $SCYLLA_IP --rpc-address $SCYLLA_IP \
+        --api-address $SCYLLA_IP \
+        --rpc-address $SCYLLA_IP \
         --listen-address $SCYLLA_IP \
+        --prometheus-address $SCYLLA_IP \
         --seed-provider-parameters seeds=$SCYLLA_IP \
         --workdir "$tmp_dir" \
         --server-encryption-options keyfile="$tmp_dir/scylla.key" \
@@ -81,7 +93,7 @@ SCYLLA_PROCESS=$!
 # test. This requires connecting to Scylla with CQL - we'll wait up for
 # one minute for this to work:
 setup_authentication() {
-    python3 -c 'from cassandra.cluster import Cluster; Cluster().connect().execute("INSERT INTO system_auth.roles (role, salted_hash) VALUES ('\''alternator'\'', '\''secret_pass'\'')")'
+    python3 -c 'from cassandra.cluster import Cluster; Cluster(["'$SCYLLA_IP'"]).connect().execute("INSERT INTO system_auth.roles (role, salted_hash) VALUES ('\''alternator'\'', '\''secret_pass'\'')")'
 }
 echo "Scylla is: $SCYLLA."
 echo -n "Booting Scylla..."
@@ -127,7 +139,8 @@ else
 fi
 
 cd "$script_path"
-pytest "$@"
+set +e
+pytest --url $alternator_url "$@"
 code=$?
 case $code in
 0) summary="Alternator tests pass";;
diff --git a/test/alternator/suite.yaml b/test/alternator/suite.yaml
new file mode 100644
index 0000000000..8919604e66
--- /dev/null
+++ b/test/alternator/suite.yaml
@@ -0,0 +1 @@
+type: Run
diff --git a/alternator-test/test_authorization.py b/test/alternator/test_authorization.py
similarity index 100%
rename from alternator-test/test_authorization.py
rename to test/alternator/test_authorization.py
diff --git a/alternator-test/test_batch.py b/test/alternator/test_batch.py
similarity index 100%
rename from alternator-test/test_batch.py
rename to test/alternator/test_batch.py
diff --git a/alternator-test/test_condition_expression.py b/test/alternator/test_condition_expression.py
similarity index 100%
rename from alternator-test/test_condition_expression.py
rename to test/alternator/test_condition_expression.py
diff --git a/alternator-test/test_describe_endpoints.py b/test/alternator/test_describe_endpoints.py
similarity index 100%
rename from alternator-test/test_describe_endpoints.py
rename to test/alternator/test_describe_endpoints.py
diff --git a/alternator-test/test_describe_table.py b/test/alternator/test_describe_table.py
similarity index 100%
rename from alternator-test/test_describe_table.py
rename to test/alternator/test_describe_table.py
diff --git a/alternator-test/test_expected.py b/test/alternator/test_expected.py
similarity index 100%
rename from alternator-test/test_expected.py
rename to test/alternator/test_expected.py
diff --git a/alternator-test/test_gsi.py b/test/alternator/test_gsi.py
similarity index 100%
rename from alternator-test/test_gsi.py
rename to test/alternator/test_gsi.py
diff --git a/alternator-test/test_health.py b/test/alternator/test_health.py
similarity index 100%
rename from alternator-test/test_health.py
rename to test/alternator/test_health.py
diff --git a/alternator-test/test_item.py b/test/alternator/test_item.py
similarity index 100%
rename from alternator-test/test_item.py
rename to test/alternator/test_item.py
diff --git a/alternator-test/test_key_condition_expression.py b/test/alternator/test_key_condition_expression.py
similarity index 100%
rename from alternator-test/test_key_condition_expression.py
rename to test/alternator/test_key_condition_expression.py
diff --git a/alternator-test/test_lsi.py b/test/alternator/test_lsi.py
similarity index 100%
rename from alternator-test/test_lsi.py
rename to test/alternator/test_lsi.py
diff --git a/alternator-test/test_manual_requests.py b/test/alternator/test_manual_requests.py
similarity index 100%
rename from alternator-test/test_manual_requests.py
rename to test/alternator/test_manual_requests.py
diff --git a/alternator-test/test_nested.py b/test/alternator/test_nested.py
similarity index 100%
rename from alternator-test/test_nested.py
rename to test/alternator/test_nested.py
diff --git a/alternator-test/test_projection_expression.py b/test/alternator/test_projection_expression.py
similarity index 100%
rename from alternator-test/test_projection_expression.py
rename to test/alternator/test_projection_expression.py
diff --git a/alternator-test/test_query.py b/test/alternator/test_query.py
similarity index 100%
rename from alternator-test/test_query.py
rename to test/alternator/test_query.py
diff --git a/alternator-test/test_returnvalues.py b/test/alternator/test_returnvalues.py
similarity index 100%
rename from alternator-test/test_returnvalues.py
rename to test/alternator/test_returnvalues.py
diff --git a/alternator-test/test_scan.py b/test/alternator/test_scan.py
similarity index 100%
rename from alternator-test/test_scan.py
rename to test/alternator/test_scan.py
diff --git a/alternator-test/test_scylla.py b/test/alternator/test_scylla.py
similarity index 100%
rename from alternator-test/test_scylla.py
rename to test/alternator/test_scylla.py
diff --git a/alternator-test/test_table.py b/test/alternator/test_table.py
similarity index 100%
rename from alternator-test/test_table.py
rename to test/alternator/test_table.py
diff --git a/alternator-test/test_tag.py b/test/alternator/test_tag.py
similarity index 100%
rename from alternator-test/test_tag.py
rename to test/alternator/test_tag.py
diff --git a/alternator-test/test_update_expression.py b/test/alternator/test_update_expression.py
similarity index 100%
rename from alternator-test/test_update_expression.py
rename to test/alternator/test_update_expression.py
diff --git a/alternator-test/util.py b/test/alternator/util.py
similarity index 100%
rename from alternator-test/util.py
rename to test/alternator/util.py