diff --git a/dist/common/scripts/scylla_io_setup b/dist/common/scripts/scylla_io_setup
old mode 100755
new mode 100644
index 600a2c237d..602690ecf2
--- a/dist/common/scripts/scylla_io_setup
+++ b/dist/common/scripts/scylla_io_setup
@@ -1,81 +1,87 @@
-#!/bin/bash
+#!/usr/bin/python
-. /usr/lib/scylla/scylla_lib.sh
+# Copyright (C) 2017 ScyllaDB
+#
+# This file is part of Scylla.
+#
+# Scylla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Scylla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Scylla. If not, see .
-print_usage() {
- echo "scylla_io_setup --ami"
- echo " --ami setup AMI instance"
- exit 1
-}
+import os
+import re
+from string import atoi
+import scylla_util
+import subprocess
+import argparse
+import yaml
+import logging
+import sys
-AMI_OPT=0
-while [ $# -gt 0 ]; do
- case "$1" in
- "--ami")
- AMI_OPT=1
- shift 1
- ;;
- *)
- print_usage
- ;;
- esac
-done
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description='IO Setup script for Scylla.')
+ parser.add_argument('--ami', dest='ami', action='store_true',
+ help='configure AWS AMI')
+ args = parser.parse_args()
+ cpudata = scylla_util.scylla_cpuinfo()
+ if not scylla_util.is_developer_mode():
+ if args.ami:
+ idata = scylla_util.aws_instance()
+ nr_io_queues = cpudata.nr_shards()
-is_developer_mode() {
- cat /etc/scylla.d/dev-mode.conf|egrep -c "\-\-developer-mode(\s+|=)(1|true)"
-}
+ if idata.instance_class() == "i3":
+ sysfs_file = "/sys/block/%s/queue/nr_requests"
+ max_seastar_shard_req = 128
+ max_sysfs_shard_req = sum([ atoi(file(sysfs_file % x).readline().strip()) for x in idata.ephemeral_disks() ])
+ # obtained running iotune multiple times against a single i3 disk.
+ max_iotune_disk = 192 * len(idata.ephemeral_disks())
+ nr_reqs = min(max_seastar_shard_req * cpudata.nr_shards(), max_sysfs_shard_req, max_iotune_disk)
+ elif idata.instance_class() == "i2":
+ nr_reqs = 32 * len(idata.ephemeral_disks())
+ else:
+ nr_reqs = 16 * max(len(idata.ephemeral_disks()), 2)
+ if nr_reqs/nr_io_queues < 4:
+ nr_io_queues = nr_reqs / 4
+ ioconf = file("/etc/scylla.d/io.conf", "w")
+ ioconf.write("SEASTAR_IO=\"--num-io-queues {} --max-io-requests {}\"\n".format(nr_io_queues, nr_reqs))
+ else:
+ if os.environ.has_key("SCYLLA_CONF"):
+ conf_dir = os.environ["SCYLLA_CONF"]
+ else:
+ conf_dir = "/etc/scylla"
+ cfg = yaml.load(open(os.path.join(conf_dir, "scylla.yaml")))
+ data_dirs = cfg["data_file_directories"]
+ if len(data_dirs) > 1:
+ logging.warn("%d data directories found. scylla_io_setup currently lacks support for it, and only %s will be evaluated",
+ len(data_dirs), data_dirs[0])
-output_to_user()
-{
- echo "$1"
- logger -p user.err "$1"
-}
+ data_dir = data_dirs[0]
+ iotune_args = []
+ if cpudata.cpuset():
+ iotune_args += [ "--cpuset", ",".join(map(str, cpudata.cpuset())) ]
+ elif cpudata.smp():
+ iotune_args += [ "--smp", cpudata.smp() ]
-if [ `is_developer_mode` -eq 0 ]; then
- SMP=`echo $CPUSET|grep smp|sed -e "s/^.*smp\(\s\+\|=\)\([^ ]*\).*$/\2/"`
- CPUSET=`echo $CPUSET|grep cpuset|sed -e "s/^.*\(--cpuset\(\s\+\|=\)[^ ]*\).*$/\1/"`
- if [ $AMI_OPT -eq 1 ]; then
- NR_CPU=`cat /proc/cpuinfo |grep processor|wc -l`
- NR_DISKS=`lsblk --list --nodeps --noheadings | grep -v xvda | grep xvd | wc -l`
- TYPE=`curl http://169.254.169.254/latest/meta-data/instance-type|cut -d . -f 1`
+ try:
+ subprocess.check_call(["iotune",
+ "--evaluation-directory", data_dir,
+ "--format", "envfile",
+ "--options-file", "/etc/scylla.d/io.conf"] + iotune_args)
+ except Exception:
+ logging.error("%s did not pass validation tests, it may not be on XFS and/or has limited disk space.\n"
+ "This is a non-supported setup, and performance is expected to be very bad.\n"
+ "For better performance, placing your data on XFS-formatted directories is required.\n"
+ "To override this error, enable developer mode as follow:\n"
+ "sudo /usr/lib/scylla/scylla_dev_mode_setup --developer-mode 1", data_dir)
+ sys.exit(1)
- if [ "$SMP" != "" ]; then
- NR_CPU=$SMP
- fi
- NR_SHARDS=$NR_CPU
- if [ $NR_CPU -ge 8 ] && [ "$SET_NIC" = "no" ]; then
- NR_SHARDS=$((NR_CPU - 1))
- fi
- if [ $NR_DISKS -lt 2 ]; then NR_DISKS=2; fi
-
- NR_REQS=$((32 * $NR_DISKS / 2))
-
- NR_IO_QUEUES=$NR_SHARDS
- if [ $(($NR_REQS/$NR_IO_QUEUES)) -lt 4 ]; then
- NR_IO_QUEUES=$(($NR_REQS / 4))
- fi
-
- NR_IO_QUEUES=$((NR_IO_QUEUES>NR_SHARDS?NR_SHARDS:NR_IO_QUEUES))
- NR_REQS=$(($(($NR_REQS / $NR_IO_QUEUES)) * $NR_IO_QUEUES))
- if [ "$TYPE" = "i2" ]; then
- NR_REQS=$(($NR_REQS * 2))
- fi
-
- echo "SEASTAR_IO=\"--num-io-queues $NR_IO_QUEUES --max-io-requests $NR_REQS\"" > /etc/scylla.d/io.conf
- else
- DATA_DIR=`/usr/lib/scylla/scylla_config_get.py --config $SCYLLA_CONF/scylla.yaml --get data_file_directories|head -n1`
- IOTUNE_ARGS="$CPUSET"
- if [ "$SMP" != "" ]; then
- IOTUNE_ARGS="$IOTUNE_ARGS --smp $SMP"
- fi
- iotune --evaluation-directory $DATA_DIR --format envfile --options-file /etc/scylla.d/io.conf $IOTUNE_ARGS
- if [ $? -ne 0 ]; then
- output_to_user "/var/lib/scylla did not pass validation tests, it may not be on XFS and/or has limited disk space."
- output_to_user "This is a non-supported setup, and performance is expected to be very bad."
- output_to_user "For better performance, placing your data on XFS-formatted directories is required."
- output_to_user "To override this error, enable developer mode as follow:"
- output_to_user " sudo /usr/lib/scylla/scylla_dev_mode_setup --developer-mode 1"
- fi
- fi
-fi
diff --git a/dist/common/scripts/scylla_util.py b/dist/common/scripts/scylla_util.py
new file mode 100644
index 0000000000..a400ab6280
--- /dev/null
+++ b/dist/common/scripts/scylla_util.py
@@ -0,0 +1,214 @@
+# Copyright (C) 2017 ScyllaDB
+
+# This file is part of Scylla.
+#
+# Scylla is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Scylla is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Scylla. If not, see .
+
+import urllib2
+import urllib
+import logging
+import time
+import re
+import os
+import string
+
+def curl(url):
+ max_retries = 5
+ retries = 0
+ while True:
+ try:
+ req = urllib2.Request(url)
+ return urllib2.urlopen(req).read()
+ except urllib2.HTTPError:
+ logging.warn("Failed to grab %s..." % url)
+ time.sleep(5)
+ retries += 1
+ if (retries >= max_retries):
+ raise
+
+class aws_instance:
+ """Describe several aspects of the current AWS instance"""
+ def __disk_name(self, dev):
+ name = re.compile(r"(?:/dev/)?(?P[a-zA-Z]+)\d*")
+ return name.search(dev).group("devname")
+
+ def __instance_metadata(self, path):
+ return curl("http://169.254.169.254/latest/meta-data/" + path)
+
+ def __device_exists(self, dev):
+ if dev[0:4] != "/dev":
+ dev = "/dev/%s" %dev
+ return os.path.exists(dev)
+
+ def __xenify(self, devname):
+ dev = self.__instance_metadata('block-device-mapping/' + devname)
+ return dev.replace("sd", "xvd")
+
+ def __populate_disks(self):
+ devmap = self.__instance_metadata("block-device-mapping")
+ self._disks = {}
+ devname = re.compile("^\D+")
+ nvme_re = re.compile(r"nvme\d+n\d+$")
+ nvmes_present = filter(nvme_re.match, os.listdir("/dev"))
+ if nvmes_present:
+ self._disks["ephemeral"] = nvmes_present;
+
+ for dev in devmap.splitlines():
+ t = devname.match(dev).group()
+ if t == "ephemeral" and nvmes_present:
+ continue;
+ if not self._disks.has_key(t):
+ self._disks[t] = []
+ if not self.__device_exists(self.__xenify(dev)):
+ continue
+ self._disks[t] += [ self.__xenify(dev) ]
+
+ def __init__(self):
+ self._type = self.__instance_metadata("instance-type")
+ self.__populate_disks()
+
+ def instance(self):
+ """Returns which instance we are running in. i.e.: i3.16xlarge"""
+ return self._type
+
+ def instance_size(self):
+ """Returns the size of the instance we are running in. i.e.: 16xlarge"""
+ return self._type.split(".")[1]
+
+ def instance_class(self):
+ """Returns the class of the instance we are running in. i.e.: i3"""
+ return self._type.split(".")[0]
+
+ def disks(self):
+ """Returns all disks in the system, as visible from the AWS registry"""
+ disks = set()
+ for v in self._disks.values():
+ disks = disks.union([ self.__disk_name(x) for x in v ])
+ return disks
+
+ def root_device(self):
+ """Returns the device being used for root data. Unlike root_disk(),
+ which will return a device name (i.e. xvda), this function will return
+ the full path to the root partition as returned by the AWS instance
+ metadata registry"""
+ return set(self._disks["root"])
+
+ def root_disk(self):
+ """Returns the disk used for the root partition"""
+ return self.__disk_name(self._disks["root"][0])
+
+ def non_root_disks(self):
+ """Returns all attached disks but root. Include ephemeral and EBS devices"""
+ return set(self._disks["ephemeral"] + self._disks["ebs"])
+
+ def ephemeral_disks(self):
+ """Returns all ephemeral disks. Include standard SSDs and NVMe"""
+ return set(self._disks["ephemeral"])
+
+ def ebs_disks(self):
+ """Returns all EBS disks"""
+ return set(self._disks["ephemeral"])
+
+ def public_ipv4(self):
+ """Returns the public IPv4 address of this instance"""
+ return self.__instance_metadata("public-ipv4")
+
+ def private_ipv4(self):
+ """Returns the private IPv4 address of this instance"""
+ return self.__instance_metadata("local-ipv4")
+
+
+## Regular expression helpers
+# non-advancing comment matcher
+_nocomment=r"^\s*(?!#)"
+# non-capturing grouping
+_scyllaeq=r"(?:\s*|=)"
+_cpuset = r"(?:\s*--cpuset" + _scyllaeq + r"(?P\d+(?:[-,]\d+)*))"
+_smp = r"(?:\s*--smp" + _scyllaeq + r"(?P\d+))"
+
+def _reopt(s):
+ return s + r"?"
+
+def is_developer_mode():
+ f = file("/etc/scylla.d/dev-mode.conf", "ro")
+ pattern = re.compile(_nocomment + r".*developer-mode" + _scyllaeq + "(1|true)")
+ return len([ x for x in f.xreadlines() if pattern.match(x) ]) >= 1
+
+class scylla_cpuinfo:
+ """Class containing information about how Scylla sees CPUs in this machine.
+ Information that can be probed include in which hyperthreads Scylla is configured
+ to run, how many total threads exist in the system, etc"""
+ def __parse_cpuset(self):
+ f = file("/etc/scylla.d/cpuset.conf", "ro")
+ pattern = re.compile(_nocomment + r"CPUSET=\s*\"" + _reopt(_cpuset) + _reopt(_smp) + "\s*\"")
+ grp = [ pattern.match(x) for x in f.readlines() if pattern.match(x) ]
+ # if more than one, use last
+ d = grp[-1].groupdict()
+ actual_set = set()
+ if d["cpuset"]:
+ groups = d["cpuset"].split(",")
+ for g in groups:
+ ends = [ string.atoi(x) for x in g.split("-") ]
+ actual_set = actual_set.union(set(xrange(ends[0], ends[-1] +1)))
+ d["cpuset"] = actual_set
+ if d["smp"]:
+ d["smp"] = atoi(d["smp"])
+ self._cpu_data = d;
+
+ def __system_cpus(self):
+ cur_proc = -1
+ f = file("/proc/cpuinfo", "ro")
+ results = {}
+ for line in f.xreadlines():
+ if line == '\n':
+ continue
+ key, value = [ x.strip() for x in line.split(":") ]
+ if key == "processor":
+ cur_proc = string.atoi(value)
+ results[cur_proc] = {}
+ results[cur_proc][key] = value
+ return results
+
+ def __init__(self):
+ self.__parse_cpuset()
+ self._cpu_data["system"] = self.__system_cpus()
+
+ def system_cpuinfo(self):
+ """Returns parsed information about CPUs in the system"""
+ return self._cpu_data["system"]
+
+ def system_nr_threads(self):
+ """Returns the number of threads available in the system"""
+ return len(self._cpu_data["system"])
+
+ def system_nr_cores(self):
+ """Returns the number of cores available in the system"""
+ return len(set([ x['core id'] for x in self._cpu_data["system"].values() ]))
+
+ def cpuset(self):
+ """Returns the current cpuset Scylla is configured to use. Returns None if no constraints exist"""
+ return self._cpu_data["cpuset"]
+
+ def smp(self):
+ """Returns the explicit smp configuration for Scylla, returns None if no constraints exist"""
+ return self._cpu_data["smp"]
+
+ def nr_shards(self):
+ """How many shards will Scylla use in this machine"""
+ if self._cpu_data["smp"]:
+ return self._cpu_data["smp"]
+ elif self._cpu_data["cpuset"]:
+ return len(self._cpu_data["cpuset"])
+ else:
+ return len(self._cpu_data["system"])