From ba7010b7a54623a57f402b0eaa2e62495e734f55 Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Mon, 3 Apr 2017 16:14:09 -0400 Subject: [PATCH 1/2] scripts: add python module with common utilities As we convert more stuff to python, we'll have more opportunities for sharing code between them. We already do that for the bash scripts with a file "scylla_lib.sh". We'll do the same for python. Signed-off-by: Glauber Costa --- dist/common/scripts/scylla_util.py | 214 +++++++++++++++++++++++++++++ 1 file changed, 214 insertions(+) create mode 100644 dist/common/scripts/scylla_util.py diff --git a/dist/common/scripts/scylla_util.py b/dist/common/scripts/scylla_util.py new file mode 100644 index 0000000000..a400ab6280 --- /dev/null +++ b/dist/common/scripts/scylla_util.py @@ -0,0 +1,214 @@ +# Copyright (C) 2017 ScyllaDB + +# This file is part of Scylla. +# +# Scylla is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Scylla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Scylla. If not, see . + +import urllib2 +import urllib +import logging +import time +import re +import os +import string + +def curl(url): + max_retries = 5 + retries = 0 + while True: + try: + req = urllib2.Request(url) + return urllib2.urlopen(req).read() + except urllib2.HTTPError: + logging.warn("Failed to grab %s..." % url) + time.sleep(5) + retries += 1 + if (retries >= max_retries): + raise + +class aws_instance: + """Describe several aspects of the current AWS instance""" + def __disk_name(self, dev): + name = re.compile(r"(?:/dev/)?(?P[a-zA-Z]+)\d*") + return name.search(dev).group("devname") + + def __instance_metadata(self, path): + return curl("http://169.254.169.254/latest/meta-data/" + path) + + def __device_exists(self, dev): + if dev[0:4] != "/dev": + dev = "/dev/%s" %dev + return os.path.exists(dev) + + def __xenify(self, devname): + dev = self.__instance_metadata('block-device-mapping/' + devname) + return dev.replace("sd", "xvd") + + def __populate_disks(self): + devmap = self.__instance_metadata("block-device-mapping") + self._disks = {} + devname = re.compile("^\D+") + nvme_re = re.compile(r"nvme\d+n\d+$") + nvmes_present = filter(nvme_re.match, os.listdir("/dev")) + if nvmes_present: + self._disks["ephemeral"] = nvmes_present; + + for dev in devmap.splitlines(): + t = devname.match(dev).group() + if t == "ephemeral" and nvmes_present: + continue; + if not self._disks.has_key(t): + self._disks[t] = [] + if not self.__device_exists(self.__xenify(dev)): + continue + self._disks[t] += [ self.__xenify(dev) ] + + def __init__(self): + self._type = self.__instance_metadata("instance-type") + self.__populate_disks() + + def instance(self): + """Returns which instance we are running in. i.e.: i3.16xlarge""" + return self._type + + def instance_size(self): + """Returns the size of the instance we are running in. i.e.: 16xlarge""" + return self._type.split(".")[1] + + def instance_class(self): + """Returns the class of the instance we are running in. i.e.: i3""" + return self._type.split(".")[0] + + def disks(self): + """Returns all disks in the system, as visible from the AWS registry""" + disks = set() + for v in self._disks.values(): + disks = disks.union([ self.__disk_name(x) for x in v ]) + return disks + + def root_device(self): + """Returns the device being used for root data. Unlike root_disk(), + which will return a device name (i.e. xvda), this function will return + the full path to the root partition as returned by the AWS instance + metadata registry""" + return set(self._disks["root"]) + + def root_disk(self): + """Returns the disk used for the root partition""" + return self.__disk_name(self._disks["root"][0]) + + def non_root_disks(self): + """Returns all attached disks but root. Include ephemeral and EBS devices""" + return set(self._disks["ephemeral"] + self._disks["ebs"]) + + def ephemeral_disks(self): + """Returns all ephemeral disks. Include standard SSDs and NVMe""" + return set(self._disks["ephemeral"]) + + def ebs_disks(self): + """Returns all EBS disks""" + return set(self._disks["ephemeral"]) + + def public_ipv4(self): + """Returns the public IPv4 address of this instance""" + return self.__instance_metadata("public-ipv4") + + def private_ipv4(self): + """Returns the private IPv4 address of this instance""" + return self.__instance_metadata("local-ipv4") + + +## Regular expression helpers +# non-advancing comment matcher +_nocomment=r"^\s*(?!#)" +# non-capturing grouping +_scyllaeq=r"(?:\s*|=)" +_cpuset = r"(?:\s*--cpuset" + _scyllaeq + r"(?P\d+(?:[-,]\d+)*))" +_smp = r"(?:\s*--smp" + _scyllaeq + r"(?P\d+))" + +def _reopt(s): + return s + r"?" + +def is_developer_mode(): + f = file("/etc/scylla.d/dev-mode.conf", "ro") + pattern = re.compile(_nocomment + r".*developer-mode" + _scyllaeq + "(1|true)") + return len([ x for x in f.xreadlines() if pattern.match(x) ]) >= 1 + +class scylla_cpuinfo: + """Class containing information about how Scylla sees CPUs in this machine. + Information that can be probed include in which hyperthreads Scylla is configured + to run, how many total threads exist in the system, etc""" + def __parse_cpuset(self): + f = file("/etc/scylla.d/cpuset.conf", "ro") + pattern = re.compile(_nocomment + r"CPUSET=\s*\"" + _reopt(_cpuset) + _reopt(_smp) + "\s*\"") + grp = [ pattern.match(x) for x in f.readlines() if pattern.match(x) ] + # if more than one, use last + d = grp[-1].groupdict() + actual_set = set() + if d["cpuset"]: + groups = d["cpuset"].split(",") + for g in groups: + ends = [ string.atoi(x) for x in g.split("-") ] + actual_set = actual_set.union(set(xrange(ends[0], ends[-1] +1))) + d["cpuset"] = actual_set + if d["smp"]: + d["smp"] = atoi(d["smp"]) + self._cpu_data = d; + + def __system_cpus(self): + cur_proc = -1 + f = file("/proc/cpuinfo", "ro") + results = {} + for line in f.xreadlines(): + if line == '\n': + continue + key, value = [ x.strip() for x in line.split(":") ] + if key == "processor": + cur_proc = string.atoi(value) + results[cur_proc] = {} + results[cur_proc][key] = value + return results + + def __init__(self): + self.__parse_cpuset() + self._cpu_data["system"] = self.__system_cpus() + + def system_cpuinfo(self): + """Returns parsed information about CPUs in the system""" + return self._cpu_data["system"] + + def system_nr_threads(self): + """Returns the number of threads available in the system""" + return len(self._cpu_data["system"]) + + def system_nr_cores(self): + """Returns the number of cores available in the system""" + return len(set([ x['core id'] for x in self._cpu_data["system"].values() ])) + + def cpuset(self): + """Returns the current cpuset Scylla is configured to use. Returns None if no constraints exist""" + return self._cpu_data["cpuset"] + + def smp(self): + """Returns the explicit smp configuration for Scylla, returns None if no constraints exist""" + return self._cpu_data["smp"] + + def nr_shards(self): + """How many shards will Scylla use in this machine""" + if self._cpu_data["smp"]: + return self._cpu_data["smp"] + elif self._cpu_data["cpuset"]: + return len(self._cpu_data["cpuset"]) + else: + return len(self._cpu_data["system"]) From 2fa698ee95a8a34fa216073577c606518f9e518e Mon Sep 17 00:00:00 2001 From: Glauber Costa Date: Tue, 4 Apr 2017 14:39:06 -0400 Subject: [PATCH 2/2] rewrite scylla_io_setup in python We do it using the new scylla_util.py library. As we do it, we also enable i3 support. Signed-off-by: Glauber Costa --- dist/common/scripts/scylla_io_setup | 152 +++++++++++++++------------- 1 file changed, 79 insertions(+), 73 deletions(-) mode change 100755 => 100644 dist/common/scripts/scylla_io_setup diff --git a/dist/common/scripts/scylla_io_setup b/dist/common/scripts/scylla_io_setup old mode 100755 new mode 100644 index 600a2c237d..602690ecf2 --- a/dist/common/scripts/scylla_io_setup +++ b/dist/common/scripts/scylla_io_setup @@ -1,81 +1,87 @@ -#!/bin/bash +#!/usr/bin/python -. /usr/lib/scylla/scylla_lib.sh +# Copyright (C) 2017 ScyllaDB +# +# This file is part of Scylla. +# +# Scylla is free software: you can redistribute it and/or modify +# it under the terms of the GNU Affero General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Scylla is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Scylla. If not, see . -print_usage() { - echo "scylla_io_setup --ami" - echo " --ami setup AMI instance" - exit 1 -} +import os +import re +from string import atoi +import scylla_util +import subprocess +import argparse +import yaml +import logging +import sys -AMI_OPT=0 -while [ $# -gt 0 ]; do - case "$1" in - "--ami") - AMI_OPT=1 - shift 1 - ;; - *) - print_usage - ;; - esac -done +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='IO Setup script for Scylla.') + parser.add_argument('--ami', dest='ami', action='store_true', + help='configure AWS AMI') + args = parser.parse_args() + cpudata = scylla_util.scylla_cpuinfo() + if not scylla_util.is_developer_mode(): + if args.ami: + idata = scylla_util.aws_instance() + nr_io_queues = cpudata.nr_shards() -is_developer_mode() { - cat /etc/scylla.d/dev-mode.conf|egrep -c "\-\-developer-mode(\s+|=)(1|true)" -} + if idata.instance_class() == "i3": + sysfs_file = "/sys/block/%s/queue/nr_requests" + max_seastar_shard_req = 128 + max_sysfs_shard_req = sum([ atoi(file(sysfs_file % x).readline().strip()) for x in idata.ephemeral_disks() ]) + # obtained running iotune multiple times against a single i3 disk. + max_iotune_disk = 192 * len(idata.ephemeral_disks()) + nr_reqs = min(max_seastar_shard_req * cpudata.nr_shards(), max_sysfs_shard_req, max_iotune_disk) + elif idata.instance_class() == "i2": + nr_reqs = 32 * len(idata.ephemeral_disks()) + else: + nr_reqs = 16 * max(len(idata.ephemeral_disks()), 2) + if nr_reqs/nr_io_queues < 4: + nr_io_queues = nr_reqs / 4 + ioconf = file("/etc/scylla.d/io.conf", "w") + ioconf.write("SEASTAR_IO=\"--num-io-queues {} --max-io-requests {}\"\n".format(nr_io_queues, nr_reqs)) + else: + if os.environ.has_key("SCYLLA_CONF"): + conf_dir = os.environ["SCYLLA_CONF"] + else: + conf_dir = "/etc/scylla" + cfg = yaml.load(open(os.path.join(conf_dir, "scylla.yaml"))) + data_dirs = cfg["data_file_directories"] + if len(data_dirs) > 1: + logging.warn("%d data directories found. scylla_io_setup currently lacks support for it, and only %s will be evaluated", + len(data_dirs), data_dirs[0]) -output_to_user() -{ - echo "$1" - logger -p user.err "$1" -} + data_dir = data_dirs[0] + iotune_args = [] + if cpudata.cpuset(): + iotune_args += [ "--cpuset", ",".join(map(str, cpudata.cpuset())) ] + elif cpudata.smp(): + iotune_args += [ "--smp", cpudata.smp() ] -if [ `is_developer_mode` -eq 0 ]; then - SMP=`echo $CPUSET|grep smp|sed -e "s/^.*smp\(\s\+\|=\)\([^ ]*\).*$/\2/"` - CPUSET=`echo $CPUSET|grep cpuset|sed -e "s/^.*\(--cpuset\(\s\+\|=\)[^ ]*\).*$/\1/"` - if [ $AMI_OPT -eq 1 ]; then - NR_CPU=`cat /proc/cpuinfo |grep processor|wc -l` - NR_DISKS=`lsblk --list --nodeps --noheadings | grep -v xvda | grep xvd | wc -l` - TYPE=`curl http://169.254.169.254/latest/meta-data/instance-type|cut -d . -f 1` + try: + subprocess.check_call(["iotune", + "--evaluation-directory", data_dir, + "--format", "envfile", + "--options-file", "/etc/scylla.d/io.conf"] + iotune_args) + except Exception: + logging.error("%s did not pass validation tests, it may not be on XFS and/or has limited disk space.\n" + "This is a non-supported setup, and performance is expected to be very bad.\n" + "For better performance, placing your data on XFS-formatted directories is required.\n" + "To override this error, enable developer mode as follow:\n" + "sudo /usr/lib/scylla/scylla_dev_mode_setup --developer-mode 1", data_dir) + sys.exit(1) - if [ "$SMP" != "" ]; then - NR_CPU=$SMP - fi - NR_SHARDS=$NR_CPU - if [ $NR_CPU -ge 8 ] && [ "$SET_NIC" = "no" ]; then - NR_SHARDS=$((NR_CPU - 1)) - fi - if [ $NR_DISKS -lt 2 ]; then NR_DISKS=2; fi - - NR_REQS=$((32 * $NR_DISKS / 2)) - - NR_IO_QUEUES=$NR_SHARDS - if [ $(($NR_REQS/$NR_IO_QUEUES)) -lt 4 ]; then - NR_IO_QUEUES=$(($NR_REQS / 4)) - fi - - NR_IO_QUEUES=$((NR_IO_QUEUES>NR_SHARDS?NR_SHARDS:NR_IO_QUEUES)) - NR_REQS=$(($(($NR_REQS / $NR_IO_QUEUES)) * $NR_IO_QUEUES)) - if [ "$TYPE" = "i2" ]; then - NR_REQS=$(($NR_REQS * 2)) - fi - - echo "SEASTAR_IO=\"--num-io-queues $NR_IO_QUEUES --max-io-requests $NR_REQS\"" > /etc/scylla.d/io.conf - else - DATA_DIR=`/usr/lib/scylla/scylla_config_get.py --config $SCYLLA_CONF/scylla.yaml --get data_file_directories|head -n1` - IOTUNE_ARGS="$CPUSET" - if [ "$SMP" != "" ]; then - IOTUNE_ARGS="$IOTUNE_ARGS --smp $SMP" - fi - iotune --evaluation-directory $DATA_DIR --format envfile --options-file /etc/scylla.d/io.conf $IOTUNE_ARGS - if [ $? -ne 0 ]; then - output_to_user "/var/lib/scylla did not pass validation tests, it may not be on XFS and/or has limited disk space." - output_to_user "This is a non-supported setup, and performance is expected to be very bad." - output_to_user "For better performance, placing your data on XFS-formatted directories is required." - output_to_user "To override this error, enable developer mode as follow:" - output_to_user " sudo /usr/lib/scylla/scylla_dev_mode_setup --developer-mode 1" - fi - fi -fi