mirrors/scylladb
1
0
Fork 0
mirror of https://github.com/scylladb/scylladb.git synced 2026-04-23 18:10:39 +00:00
Code Issues Packages Projects Releases Wiki Activity
Files
ecf854affcab4ffe0cd7fca3dfb2472f7c635fdf
scylladb/dist/common/scripts/scylla_io_setup

345 lines
17 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# Copyright (C) 2017-present ScyllaDB
#
#
# This file is part of Scylla.
#
# Scylla is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# Scylla is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
import os
import re
from scylla_util import *
import subprocess
import argparse
import yaml
import logging
import sys
import scylla_blocktune as blocktune
# Regular expression helpers
# non-advancing comment matcher
_nocomment = r"^\s*(?!#)"
# non-capturing grouping
_scyllaeq = r"(?:\s*|=)"
_cpuset = r"(?:\s*--cpuset" + _scyllaeq + r"(?P<cpuset>\d+(?:[-,]\d+)*))"
_smp = r"(?:\s*--smp" + _scyllaeq + r"(?P<smp>\d+))"
def _reopt(s):
return s + r"?"
def is_developer_mode():
f = open(etcdir() + "/scylla.d/dev-mode.conf", "r")
pattern = re.compile(_nocomment + r".*developer-mode" + _scyllaeq + "(1|true)")
return len([x for x in f if pattern.match(x)]) >= 1
class scylla_cpuinfo:
"""Class containing information about how Scylla sees CPUs in this machine.
Information that can be probed include in which hyperthreads Scylla is configured
to run, how many total threads exist in the system, etc"""
def __parse_cpuset(self):
f = open(etcdir() + "/scylla.d/cpuset.conf", "r")
pattern = re.compile(_nocomment + r"CPUSET=\s*\"" + _reopt(_cpuset) + _reopt(_smp) + "\s*\"")
grp = [pattern.match(x) for x in f.readlines() if pattern.match(x)]
if not grp:
d = {"cpuset": None, "smp": None}
else:
# if more than one, use last
d = grp[-1].groupdict()
actual_set = set()
if d["cpuset"]:
groups = d["cpuset"].split(",")
for g in groups:
ends = [int(x) for x in g.split("-")]
actual_set = actual_set.union(set(range(ends[0], ends[-1] + 1)))
d["cpuset"] = actual_set
if d["smp"]:
d["smp"] = int(d["smp"])
self._cpu_data = d
def __system_cpus(self):
cur_proc = -1
f = open("/proc/cpuinfo", "r")
results = {}
for line in f:
if line == '\n':
continue
key, value = [x.strip() for x in line.split(":")]
if key == "processor":
cur_proc = int(value)
results[cur_proc] = {}
results[cur_proc][key] = value
return results
def __init__(self):
self.__parse_cpuset()
self._cpu_data["system"] = self.__system_cpus()
def system_cpuinfo(self):
"""Returns parsed information about CPUs in the system"""
return self._cpu_data["system"]
def system_nr_threads(self):
"""Returns the number of threads available in the system"""
return len(self._cpu_data["system"])
def system_nr_cores(self):
"""Returns the number of cores available in the system"""
return len(set([x['core id'] for x in list(self._cpu_data["system"].values())]))
def cpuset(self):
"""Returns the current cpuset Scylla is configured to use. Returns None if no constraints exist"""
return self._cpu_data["cpuset"]
def smp(self):
"""Returns the explicit smp configuration for Scylla, returns None if no constraints exist"""
return self._cpu_data["smp"]
def nr_shards(self):
"""How many shards will Scylla use in this machine"""
if self._cpu_data["smp"]:
return self._cpu_data["smp"]
elif self._cpu_data["cpuset"]:
return len(self._cpu_data["cpuset"])
else:
return len(self._cpu_data["system"])
def run_iotune():
if "SCYLLA_CONF" in os.environ:
conf_dir = os.environ["SCYLLA_CONF"]
else:
conf_dir = etcdir() + "/scylla"
cfg = yaml.safe_load(open(os.path.join(conf_dir, "scylla.yaml")))
default_path = cfg.get('workdir') or datadir()
if not "data_file_directories" in cfg:
cfg["data_file_directories"] = [os.path.join(default_path, 'data')]
data_dirs = cfg["data_file_directories"]
for t in [ "commitlog", "hints", "view_hints", "saved_caches" ]:
key = "%s_directory" % t
if key in cfg:
data_dirs += [ cfg[key] ]
elif os.path.isdir(os.path.join(default_path, t)):
data_dirs += [ os.path.join(default_path, t) ]
iotune_args = []
for data_dir in data_dirs:
if os.path.exists(data_dir) == False:
logging.error("%s was not found. Please check the configuration and run scylla_io_setup again.\n", data_dir)
sys.exit(1)
if os.path.isdir(data_dir) == False:
logging.error("%s is not a directory. Please check the configuration and run scylla_io_setup again.\n", data_dir)
sys.exit(1)
st = os.statvfs(data_dir)
avail = st.f_bavail * st.f_frsize
rec = 10000000000
if avail < rec:
logging.error("Filesystem at %s has only %d bytes available; that is less than the recommended 10 GB. Please free up space and run scylla_io_setup again.\n", data_dir, avail)
sys.exit(1)
blocktune.tune_fs(data_dir, '2')
iotune_args += [ "--evaluation-directory", data_dir ]
if cpudata.cpuset():
iotune_args += [ "--cpuset", ",".join(map(str, cpudata.cpuset())) ]
elif cpudata.smp():
iotune_args += [ "--smp", str(cpudata.smp()) ]
try:
subprocess.check_call([bindir() + "/iotune",
"--format", "envfile",
"--options-file", etcdir() + "/scylla.d/io.conf",
"--properties-file", etcdir() + "/scylla.d/io_properties.yaml"] + iotune_args)
except Exception as e:
logging.error(e)
logging.error("%s did not pass validation tests, it may not be on XFS and/or has limited disk space.\n"
"This is a non-supported setup, and performance is expected to be very bad.\n"
"For better performance, placing your data on XFS-formatted directories is required.\n"
"To override this error, enable developer mode as follow:\n"
"sudo %s/scylla_dev_mode_setup --developer-mode 1", data_dirs, scriptsdir())
sys.exit(1)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description='IO Setup script for Scylla.')
parser.add_argument('--ami', dest='ami', action='store_true',
help='configure AWS AMI')
args = parser.parse_args()
cpudata = scylla_cpuinfo()
if not is_developer_mode():
if args.ami:
idata = aws_instance()
if not idata.is_supported_instance_class():
logging.error('{} is not supported instance type, run "scylla_io_setup" again without --ami option.'.format(idata.instance()))
sys.exit(1)
disk_properties = {}
disk_properties["mountpoint"] = datadir()
nr_disks = len(idata.ephemeral_disks())
## both i3 and i2 can run with 1 I/O Queue per shard
if idata.instance() == "i3.large":
disk_properties["read_iops"] = 111000
disk_properties["read_bandwidth"] = 653925080
disk_properties["write_iops"] = 36800
disk_properties["write_bandwidth"] = 215066473
elif idata.instance() == "i3.xlarge":
disk_properties["read_iops"] = 200800
disk_properties["read_bandwidth"] = 1185106376
disk_properties["write_iops"] = 53180
disk_properties["write_bandwidth"] = 423621267
elif idata.instance_class() == "i3":
disk_properties["read_iops"] = 411200 * nr_disks
disk_properties["read_bandwidth"] = 2015342735 * nr_disks
disk_properties["write_iops"] = 181500 * nr_disks
disk_properties["write_bandwidth"] = 808775652 * nr_disks
elif idata.instance_class() == "i3en":
if idata.instance() == "i3en.large":
disk_properties["read_iops"] = 43315
disk_properties["read_bandwidth"] = 330301440
disk_properties["write_iops"] = 33177
disk_properties["write_bandwidth"] = 165675008
elif idata.instance() in ("i3.xlarge", "i3en.2xlarge"):
disk_properties["read_iops"] = 84480 * nr_disks
disk_properties["read_bandwidth"] = 666894336 * nr_disks
disk_properties["write_iops"] = 66969 * nr_disks
disk_properties["write_bandwidth"] = 333447168 * nr_disks
else:
disk_properties["read_iops"] = 257024 * nr_disks
disk_properties["read_bandwidth"] = 2043674624 * nr_disks
disk_properties["write_iops"] = 174080 * nr_disks
disk_properties["write_bandwidth"] = 1024458752 * nr_disks
elif idata.instance_class() == "i2":
disk_properties["read_iops"] = 64000 * nr_disks
disk_properties["read_bandwidth"] = 507338935 * nr_disks
disk_properties["write_iops"] = 57100 * nr_disks
disk_properties["write_bandwidth"] = 483141731 * nr_disks
properties_file = open(etcdir() + "/scylla.d/io_properties.yaml", "w")
yaml.dump({ "disks": [ disk_properties ] }, properties_file, default_flow_style=False)
ioconf = open(etcdir() + "/scylla.d/io.conf", "w")
ioconf.write("SEASTAR_IO=\"--io-properties-file={}\"\n".format(properties_file.name))
elif gcp_instance().is_gce_instance():
idata = gcp_instance()
if idata.is_recommended_instance():
disk_properties = {}
disk_properties["mountpoint"] = datadir()
nr_disks = idata.nvmeDiskCount
# below is based on https://cloud.google.com/compute/docs/disks/local-ssd#performance
# and https://cloud.google.com/compute/docs/disks/local-ssd#nvme
# note that scylla iotune might measure more, this is GCP recommended
mbs=1024*1024
if nr_disks >= 1 and nr_disks < 4:
disk_properties["read_iops"] = 170000 * nr_disks
disk_properties["read_bandwidth"] = 660 * mbs * nr_disks
disk_properties["write_iops"] = 90000 * nr_disks
disk_properties["write_bandwidth"] = 350 * mbs * nr_disks
elif nr_disks >= 4 and nr_disks <= 8:
disk_properties["read_iops"] = 680000
disk_properties["read_bandwidth"] = 2650 * mbs
disk_properties["write_iops"] = 360000
disk_properties["write_bandwidth"] = 1400 * mbs
elif nr_disks == "16":
disk_properties["read_iops"] = 1600000
disk_properties["read_bandwidth"] = 4521251328
#below is google, above is our measured
#disk_properties["read_bandwidth"] = 6240 * mbs
disk_properties["write_iops"] = 800000
disk_properties["write_bandwidth"] = 2759452672
#below is google, above is our measured
#disk_properties["write_bandwidth"] = 3120 * mbs
elif nr_disks == "24":
disk_properties["read_iops"] = 2400000
disk_properties["read_bandwidth"] = 5921532416
#below is google, above is our measured
#disk_properties["read_bandwidth"] = 9360 * mbs
disk_properties["write_iops"] = 1200000
disk_properties["write_bandwidth"] = 4663037952
#below is google, above is our measured
#disk_properties["write_bandwidth"] = 4680 * mbs
if "read_iops" in disk_properties:
properties_file = open(etcdir() + "/scylla.d/io_properties.yaml", "w")
yaml.dump({"disks": [disk_properties]}, properties_file, default_flow_style=False)
ioconf = open(etcdir() + "/scylla.d/io.conf", "w")
ioconf.write("SEASTAR_IO=\"--io-properties-file={}\"\n".format(properties_file.name))
else:
logging.error(
'Did not detect number of disks in Google Cloud instance setup for auto local disk tuning, running manual iotune.')
run_iotune()
else:
logging.error(
'This is not a recommended Google Cloud instance setup for auto local disk tuning, running manual iotune.')
run_iotune()
elif azure_instance().is_azure_instance():
idata = azure_instance()
if idata.is_recommended_instance():
disk_properties = {}
disk_properties["mountpoint"] = datadir()
nr_disks = idata.nvmeDiskCount
# below is based on https://docs.microsoft.com/en-us/azure/virtual-machines/lsv2-series
# note that scylla iotune might measure more, this is Azure recommended
# since write properties are not defined, they come from our iotune tests
mbs = 1024*1024
if nr_disks == 1:
disk_properties["read_iops"] = 400000
disk_properties["read_bandwidth"] = 2000 * mbs
disk_properties["write_iops"] = 271696
disk_properties["write_bandwidth"] = 1314 * mbs
elif nr_disks == 2:
disk_properties["read_iops"] = 800000
disk_properties["read_bandwidth"] = 4000 * mbs
disk_properties["write_iops"] = 552434
disk_properties["write_bandwidth"] = 2478 * mbs
elif nr_disks == 4:
disk_properties["read_iops"] = 1500000
disk_properties["read_bandwidth"] = 8000 * mbs
disk_properties["write_iops"] = 1105063
disk_properties["write_bandwidth"] = 4948 * mbs
elif nr_disks == 6:
disk_properties["read_iops"] = 2200000
disk_properties["read_bandwidth"] = 14000 * mbs
disk_properties["write_iops"] = 1616847
disk_properties["write_bandwidth"] = 7892 * mbs
elif nr_disks == 8:
disk_properties["read_iops"] = 2900000
disk_properties["read_bandwidth"] = 16000 * mbs
disk_properties["write_iops"] = 2208081
disk_properties["write_bandwidth"] = 9694 * mbs
elif nr_disks == 10:
disk_properties["read_iops"] = 3800000
disk_properties["read_bandwidth"] = 20000 * mbs
disk_properties["write_iops"] = 2546511
disk_properties["write_bandwidth"] = 11998 * mbs
if "read_iops" in disk_properties:
properties_file = open(etcdir() + "/scylla.d/io_properties.yaml", "w")
yaml.dump({"disks": [disk_properties]}, properties_file, default_flow_style=False)
ioconf = open(etcdir() + "/scylla.d/io.conf", "w")
ioconf.write("SEASTAR_IO=\"--io-properties-file={}\"\n".format(properties_file.name))
else:
logging.error(
'Did not detect number of disks in Azure Cloud instance setup for auto local disk tuning, running manual iotune.')
run_iotune()
else:
logging.error(
'This is not a recommended Azure Cloud instance setup for auto local disk tuning, running manual iotune.')
run_iotune()
else:
run_iotune()
os.chmod(etcdir() + '/scylla.d/io_properties.yaml', 0o644)
os.chmod(etcdir() + '/scylla.d/io.conf', 0o644)
Reference in New Issue View Git Blame Copy Permalink