mirror of
https://github.com/scylladb/scylladb.git
synced 2026-05-29 11:10:40 +00:00
Merge "Rewrite scylla_io_setup in python" from Glauber
"Also, this new version supports i3. Number of requests for i3 is obtained similarly as to i2: I have run tests for a single disk, and then we'll take the amount of disks into account. Other possible limits are also taken into account, like the max per-shard seastar limit of 128 in-flight request, and the per-disk limit obtained by sysfs." * 'python-io-setup' of https://github.com/glommer/scylla: rewrite scylla_io_setup in python scripts: add python module with common utilities
This commit is contained in:
Avi Kivity
152
dist/common/scripts/scylla_io_setup
vendored
Executable file → Normal file
152
dist/common/scripts/scylla_io_setup
vendored
Executable file → Normal file
@@ -1,81 +1,87 @@
|
||||
#!/bin/bash
|
||||
#!/usr/bin/python
|
||||
|
||||
. /usr/lib/scylla/scylla_lib.sh
|
||||
# Copyright (C) 2017 ScyllaDB
|
||||
#
|
||||
# This file is part of Scylla.
|
||||
#
|
||||
# Scylla is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Scylla is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
print_usage() {
|
||||
echo "scylla_io_setup --ami"
|
||||
echo " --ami setup AMI instance"
|
||||
exit 1
|
||||
}
|
||||
import os
|
||||
import re
|
||||
from string import atoi
|
||||
import scylla_util
|
||||
import subprocess
|
||||
import argparse
|
||||
import yaml
|
||||
import logging
|
||||
import sys
|
||||
|
||||
AMI_OPT=0
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
"--ami")
|
||||
AMI_OPT=1
|
||||
shift 1
|
||||
;;
|
||||
*)
|
||||
print_usage
|
||||
;;
|
||||
esac
|
||||
done
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(description='IO Setup script for Scylla.')
|
||||
parser.add_argument('--ami', dest='ami', action='store_true',
|
||||
help='configure AWS AMI')
|
||||
args = parser.parse_args()
|
||||
|
||||
cpudata = scylla_util.scylla_cpuinfo()
|
||||
if not scylla_util.is_developer_mode():
|
||||
if args.ami:
|
||||
idata = scylla_util.aws_instance()
|
||||
nr_io_queues = cpudata.nr_shards()
|
||||
|
||||
is_developer_mode() {
|
||||
cat /etc/scylla.d/dev-mode.conf|egrep -c "\-\-developer-mode(\s+|=)(1|true)"
|
||||
}
|
||||
if idata.instance_class() == "i3":
|
||||
sysfs_file = "/sys/block/%s/queue/nr_requests"
|
||||
max_seastar_shard_req = 128
|
||||
max_sysfs_shard_req = sum([ atoi(file(sysfs_file % x).readline().strip()) for x in idata.ephemeral_disks() ])
|
||||
# obtained running iotune multiple times against a single i3 disk.
|
||||
max_iotune_disk = 192 * len(idata.ephemeral_disks())
|
||||
nr_reqs = min(max_seastar_shard_req * cpudata.nr_shards(), max_sysfs_shard_req, max_iotune_disk)
|
||||
elif idata.instance_class() == "i2":
|
||||
nr_reqs = 32 * len(idata.ephemeral_disks())
|
||||
else:
|
||||
nr_reqs = 16 * max(len(idata.ephemeral_disks()), 2)
|
||||
if nr_reqs/nr_io_queues < 4:
|
||||
nr_io_queues = nr_reqs / 4
|
||||
ioconf = file("/etc/scylla.d/io.conf", "w")
|
||||
ioconf.write("SEASTAR_IO=\"--num-io-queues {} --max-io-requests {}\"\n".format(nr_io_queues, nr_reqs))
|
||||
else:
|
||||
if os.environ.has_key("SCYLLA_CONF"):
|
||||
conf_dir = os.environ["SCYLLA_CONF"]
|
||||
else:
|
||||
conf_dir = "/etc/scylla"
|
||||
cfg = yaml.load(open(os.path.join(conf_dir, "scylla.yaml")))
|
||||
data_dirs = cfg["data_file_directories"]
|
||||
if len(data_dirs) > 1:
|
||||
logging.warn("%d data directories found. scylla_io_setup currently lacks support for it, and only %s will be evaluated",
|
||||
len(data_dirs), data_dirs[0])
|
||||
|
||||
output_to_user()
|
||||
{
|
||||
echo "$1"
|
||||
logger -p user.err "$1"
|
||||
}
|
||||
data_dir = data_dirs[0]
|
||||
iotune_args = []
|
||||
if cpudata.cpuset():
|
||||
iotune_args += [ "--cpuset", ",".join(map(str, cpudata.cpuset())) ]
|
||||
elif cpudata.smp():
|
||||
iotune_args += [ "--smp", cpudata.smp() ]
|
||||
|
||||
if [ `is_developer_mode` -eq 0 ]; then
|
||||
SMP=`echo $CPUSET|grep smp|sed -e "s/^.*smp\(\s\+\|=\)\([^ ]*\).*$/\2/"`
|
||||
CPUSET=`echo $CPUSET|grep cpuset|sed -e "s/^.*\(--cpuset\(\s\+\|=\)[^ ]*\).*$/\1/"`
|
||||
if [ $AMI_OPT -eq 1 ]; then
|
||||
NR_CPU=`cat /proc/cpuinfo |grep processor|wc -l`
|
||||
NR_DISKS=`lsblk --list --nodeps --noheadings | grep -v xvda | grep xvd | wc -l`
|
||||
TYPE=`curl http://169.254.169.254/latest/meta-data/instance-type|cut -d . -f 1`
|
||||
try:
|
||||
subprocess.check_call(["iotune",
|
||||
"--evaluation-directory", data_dir,
|
||||
"--format", "envfile",
|
||||
"--options-file", "/etc/scylla.d/io.conf"] + iotune_args)
|
||||
except Exception:
|
||||
logging.error("%s did not pass validation tests, it may not be on XFS and/or has limited disk space.\n"
|
||||
"This is a non-supported setup, and performance is expected to be very bad.\n"
|
||||
"For better performance, placing your data on XFS-formatted directories is required.\n"
|
||||
"To override this error, enable developer mode as follow:\n"
|
||||
"sudo /usr/lib/scylla/scylla_dev_mode_setup --developer-mode 1", data_dir)
|
||||
sys.exit(1)
|
||||
|
||||
if [ "$SMP" != "" ]; then
|
||||
NR_CPU=$SMP
|
||||
fi
|
||||
NR_SHARDS=$NR_CPU
|
||||
if [ $NR_CPU -ge 8 ] && [ "$SET_NIC" = "no" ]; then
|
||||
NR_SHARDS=$((NR_CPU - 1))
|
||||
fi
|
||||
if [ $NR_DISKS -lt 2 ]; then NR_DISKS=2; fi
|
||||
|
||||
NR_REQS=$((32 * $NR_DISKS / 2))
|
||||
|
||||
NR_IO_QUEUES=$NR_SHARDS
|
||||
if [ $(($NR_REQS/$NR_IO_QUEUES)) -lt 4 ]; then
|
||||
NR_IO_QUEUES=$(($NR_REQS / 4))
|
||||
fi
|
||||
|
||||
NR_IO_QUEUES=$((NR_IO_QUEUES>NR_SHARDS?NR_SHARDS:NR_IO_QUEUES))
|
||||
NR_REQS=$(($(($NR_REQS / $NR_IO_QUEUES)) * $NR_IO_QUEUES))
|
||||
if [ "$TYPE" = "i2" ]; then
|
||||
NR_REQS=$(($NR_REQS * 2))
|
||||
fi
|
||||
|
||||
echo "SEASTAR_IO=\"--num-io-queues $NR_IO_QUEUES --max-io-requests $NR_REQS\"" > /etc/scylla.d/io.conf
|
||||
else
|
||||
DATA_DIR=`/usr/lib/scylla/scylla_config_get.py --config $SCYLLA_CONF/scylla.yaml --get data_file_directories|head -n1`
|
||||
IOTUNE_ARGS="$CPUSET"
|
||||
if [ "$SMP" != "" ]; then
|
||||
IOTUNE_ARGS="$IOTUNE_ARGS --smp $SMP"
|
||||
fi
|
||||
iotune --evaluation-directory $DATA_DIR --format envfile --options-file /etc/scylla.d/io.conf $IOTUNE_ARGS
|
||||
if [ $? -ne 0 ]; then
|
||||
output_to_user "/var/lib/scylla did not pass validation tests, it may not be on XFS and/or has limited disk space."
|
||||
output_to_user "This is a non-supported setup, and performance is expected to be very bad."
|
||||
output_to_user "For better performance, placing your data on XFS-formatted directories is required."
|
||||
output_to_user "To override this error, enable developer mode as follow:"
|
||||
output_to_user " sudo /usr/lib/scylla/scylla_dev_mode_setup --developer-mode 1"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
214
dist/common/scripts/scylla_util.py
vendored
Normal file
214
dist/common/scripts/scylla_util.py
vendored
Normal file
@@ -0,0 +1,214 @@
|
||||
# Copyright (C) 2017 ScyllaDB
|
||||
|
||||
# This file is part of Scylla.
|
||||
#
|
||||
# Scylla is free software: you can redistribute it and/or modify
|
||||
# it under the terms of the GNU Affero General Public License as published by
|
||||
# the Free Software Foundation, either version 3 of the License, or
|
||||
# (at your option) any later version.
|
||||
#
|
||||
# Scylla is distributed in the hope that it will be useful,
|
||||
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
# GNU General Public License for more details.
|
||||
#
|
||||
# You should have received a copy of the GNU General Public License
|
||||
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import urllib2
|
||||
import urllib
|
||||
import logging
|
||||
import time
|
||||
import re
|
||||
import os
|
||||
import string
|
||||
|
||||
def curl(url):
|
||||
max_retries = 5
|
||||
retries = 0
|
||||
while True:
|
||||
try:
|
||||
req = urllib2.Request(url)
|
||||
return urllib2.urlopen(req).read()
|
||||
except urllib2.HTTPError:
|
||||
logging.warn("Failed to grab %s..." % url)
|
||||
time.sleep(5)
|
||||
retries += 1
|
||||
if (retries >= max_retries):
|
||||
raise
|
||||
|
||||
class aws_instance:
|
||||
"""Describe several aspects of the current AWS instance"""
|
||||
def __disk_name(self, dev):
|
||||
name = re.compile(r"(?:/dev/)?(?P<devname>[a-zA-Z]+)\d*")
|
||||
return name.search(dev).group("devname")
|
||||
|
||||
def __instance_metadata(self, path):
|
||||
return curl("http://169.254.169.254/latest/meta-data/" + path)
|
||||
|
||||
def __device_exists(self, dev):
|
||||
if dev[0:4] != "/dev":
|
||||
dev = "/dev/%s" %dev
|
||||
return os.path.exists(dev)
|
||||
|
||||
def __xenify(self, devname):
|
||||
dev = self.__instance_metadata('block-device-mapping/' + devname)
|
||||
return dev.replace("sd", "xvd")
|
||||
|
||||
def __populate_disks(self):
|
||||
devmap = self.__instance_metadata("block-device-mapping")
|
||||
self._disks = {}
|
||||
devname = re.compile("^\D+")
|
||||
nvme_re = re.compile(r"nvme\d+n\d+$")
|
||||
nvmes_present = filter(nvme_re.match, os.listdir("/dev"))
|
||||
if nvmes_present:
|
||||
self._disks["ephemeral"] = nvmes_present;
|
||||
|
||||
for dev in devmap.splitlines():
|
||||
t = devname.match(dev).group()
|
||||
if t == "ephemeral" and nvmes_present:
|
||||
continue;
|
||||
if not self._disks.has_key(t):
|
||||
self._disks[t] = []
|
||||
if not self.__device_exists(self.__xenify(dev)):
|
||||
continue
|
||||
self._disks[t] += [ self.__xenify(dev) ]
|
||||
|
||||
def __init__(self):
|
||||
self._type = self.__instance_metadata("instance-type")
|
||||
self.__populate_disks()
|
||||
|
||||
def instance(self):
|
||||
"""Returns which instance we are running in. i.e.: i3.16xlarge"""
|
||||
return self._type
|
||||
|
||||
def instance_size(self):
|
||||
"""Returns the size of the instance we are running in. i.e.: 16xlarge"""
|
||||
return self._type.split(".")[1]
|
||||
|
||||
def instance_class(self):
|
||||
"""Returns the class of the instance we are running in. i.e.: i3"""
|
||||
return self._type.split(".")[0]
|
||||
|
||||
def disks(self):
|
||||
"""Returns all disks in the system, as visible from the AWS registry"""
|
||||
disks = set()
|
||||
for v in self._disks.values():
|
||||
disks = disks.union([ self.__disk_name(x) for x in v ])
|
||||
return disks
|
||||
|
||||
def root_device(self):
|
||||
"""Returns the device being used for root data. Unlike root_disk(),
|
||||
which will return a device name (i.e. xvda), this function will return
|
||||
the full path to the root partition as returned by the AWS instance
|
||||
metadata registry"""
|
||||
return set(self._disks["root"])
|
||||
|
||||
def root_disk(self):
|
||||
"""Returns the disk used for the root partition"""
|
||||
return self.__disk_name(self._disks["root"][0])
|
||||
|
||||
def non_root_disks(self):
|
||||
"""Returns all attached disks but root. Include ephemeral and EBS devices"""
|
||||
return set(self._disks["ephemeral"] + self._disks["ebs"])
|
||||
|
||||
def ephemeral_disks(self):
|
||||
"""Returns all ephemeral disks. Include standard SSDs and NVMe"""
|
||||
return set(self._disks["ephemeral"])
|
||||
|
||||
def ebs_disks(self):
|
||||
"""Returns all EBS disks"""
|
||||
return set(self._disks["ephemeral"])
|
||||
|
||||
def public_ipv4(self):
|
||||
"""Returns the public IPv4 address of this instance"""
|
||||
return self.__instance_metadata("public-ipv4")
|
||||
|
||||
def private_ipv4(self):
|
||||
"""Returns the private IPv4 address of this instance"""
|
||||
return self.__instance_metadata("local-ipv4")
|
||||
|
||||
|
||||
## Regular expression helpers
|
||||
# non-advancing comment matcher
|
||||
_nocomment=r"^\s*(?!#)"
|
||||
# non-capturing grouping
|
||||
_scyllaeq=r"(?:\s*|=)"
|
||||
_cpuset = r"(?:\s*--cpuset" + _scyllaeq + r"(?P<cpuset>\d+(?:[-,]\d+)*))"
|
||||
_smp = r"(?:\s*--smp" + _scyllaeq + r"(?P<smp>\d+))"
|
||||
|
||||
def _reopt(s):
|
||||
return s + r"?"
|
||||
|
||||
def is_developer_mode():
|
||||
f = file("/etc/scylla.d/dev-mode.conf", "ro")
|
||||
pattern = re.compile(_nocomment + r".*developer-mode" + _scyllaeq + "(1|true)")
|
||||
return len([ x for x in f.xreadlines() if pattern.match(x) ]) >= 1
|
||||
|
||||
class scylla_cpuinfo:
|
||||
"""Class containing information about how Scylla sees CPUs in this machine.
|
||||
Information that can be probed include in which hyperthreads Scylla is configured
|
||||
to run, how many total threads exist in the system, etc"""
|
||||
def __parse_cpuset(self):
|
||||
f = file("/etc/scylla.d/cpuset.conf", "ro")
|
||||
pattern = re.compile(_nocomment + r"CPUSET=\s*\"" + _reopt(_cpuset) + _reopt(_smp) + "\s*\"")
|
||||
grp = [ pattern.match(x) for x in f.readlines() if pattern.match(x) ]
|
||||
# if more than one, use last
|
||||
d = grp[-1].groupdict()
|
||||
actual_set = set()
|
||||
if d["cpuset"]:
|
||||
groups = d["cpuset"].split(",")
|
||||
for g in groups:
|
||||
ends = [ string.atoi(x) for x in g.split("-") ]
|
||||
actual_set = actual_set.union(set(xrange(ends[0], ends[-1] +1)))
|
||||
d["cpuset"] = actual_set
|
||||
if d["smp"]:
|
||||
d["smp"] = atoi(d["smp"])
|
||||
self._cpu_data = d;
|
||||
|
||||
def __system_cpus(self):
|
||||
cur_proc = -1
|
||||
f = file("/proc/cpuinfo", "ro")
|
||||
results = {}
|
||||
for line in f.xreadlines():
|
||||
if line == '\n':
|
||||
continue
|
||||
key, value = [ x.strip() for x in line.split(":") ]
|
||||
if key == "processor":
|
||||
cur_proc = string.atoi(value)
|
||||
results[cur_proc] = {}
|
||||
results[cur_proc][key] = value
|
||||
return results
|
||||
|
||||
def __init__(self):
|
||||
self.__parse_cpuset()
|
||||
self._cpu_data["system"] = self.__system_cpus()
|
||||
|
||||
def system_cpuinfo(self):
|
||||
"""Returns parsed information about CPUs in the system"""
|
||||
return self._cpu_data["system"]
|
||||
|
||||
def system_nr_threads(self):
|
||||
"""Returns the number of threads available in the system"""
|
||||
return len(self._cpu_data["system"])
|
||||
|
||||
def system_nr_cores(self):
|
||||
"""Returns the number of cores available in the system"""
|
||||
return len(set([ x['core id'] for x in self._cpu_data["system"].values() ]))
|
||||
|
||||
def cpuset(self):
|
||||
"""Returns the current cpuset Scylla is configured to use. Returns None if no constraints exist"""
|
||||
return self._cpu_data["cpuset"]
|
||||
|
||||
def smp(self):
|
||||
"""Returns the explicit smp configuration for Scylla, returns None if no constraints exist"""
|
||||
return self._cpu_data["smp"]
|
||||
|
||||
def nr_shards(self):
|
||||
"""How many shards will Scylla use in this machine"""
|
||||
if self._cpu_data["smp"]:
|
||||
return self._cpu_data["smp"]
|
||||
elif self._cpu_data["cpuset"]:
|
||||
return len(self._cpu_data["cpuset"])
|
||||
else:
|
||||
return len(self._cpu_data["system"])
|
||||
Reference in New Issue
Block a user