f1d73d0c13
" It is well known that seastar applications, like Scylla, do not play well with external processes: CPU usage from external processes may confuse the I/O and CPU schedulers and create stalls. We have also recently seen that memory usage from other application's anonymous and page cache memory can bring the system to OOM. Linux has a very good infrastructure for resource control contributed by amazingly bright engineers in the form of cgroup controllers. This infrastructure is exposed by SystemD in the form of slices: a hierarchical structure to which controllers can be attached. In true systemd way, the hierarchy is implicit in the filenames of the slice files. a "-" symbol defines the hierarchy, so the files that this patch presents, scylla-server and scylla-helper, essentially create a "scylla" cgroup at the top level with "server" and "helper" children. Later we mark the Services needed to run scylla as belonging to one or the other through the Slice= directive. Scylla DBAs can benefit from this setup by using the systemd-run utility to fire ad-hoc commands. Let's say for example that someone wants to hypothetically run a backup and transfer files to an external object store like S3, making sure that the amount of page cache used won't create swap pressure leading to database timeouts. One can then run something like: sudo systemd-run --uid=id -u scylla --gid=id -g scylla -t --slice=scylla-helper.slice /path/to/my/magical_backup_tool (or even better, the backup tool can itself be a systemd timer) " * 'slices' of https://github.com/glommer/scylla: systemd: put scylla processes in systemd slices. move postinst steps to an external script
134 lines
4.7 KiB
Python
Executable File
134 lines
4.7 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2018 ScyllaDB
|
|
#
|
|
|
|
#
|
|
# This file is part of Scylla.
|
|
#
|
|
# Scylla is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Scylla is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
|
|
import argparse
|
|
import io
|
|
import os
|
|
import subprocess
|
|
import tarfile
|
|
import pathlib
|
|
|
|
|
|
def ldd(executable):
|
|
'''Given an executable file, return a dictionary with the keys
|
|
containing its shared library dependencies and the values pointing
|
|
at the files they resolve to. A fake key ld.so points at the
|
|
dynamic loader.'''
|
|
libraries = {}
|
|
for ldd_line in subprocess.check_output(
|
|
['ldd', executable],
|
|
universal_newlines=True).splitlines():
|
|
elements = ldd_line.split()
|
|
if ldd_line.endswith('not found'):
|
|
raise Exception('ldd could not resolve {}'.format(elements[0]))
|
|
if elements[1] != '=>':
|
|
if elements[0].startswith('linux-vdso.so'):
|
|
# provided by kernel
|
|
continue
|
|
libraries['ld.so'] = os.path.realpath(elements[0])
|
|
else:
|
|
libraries[elements[0]] = os.path.realpath(elements[2])
|
|
return libraries
|
|
|
|
|
|
ap = argparse.ArgumentParser(description='Create a relocatable scylla package.')
|
|
ap.add_argument('dest',
|
|
help='Destination file (tar format)')
|
|
ap.add_argument('--mode', dest='mode', default='release',
|
|
help='Build mode (debug/release) to use')
|
|
|
|
args = ap.parse_args()
|
|
|
|
executables = ['build/{}/scylla'.format(args.mode),
|
|
'build/{}/iotune'.format(args.mode),
|
|
'/usr/bin/patchelf',
|
|
'/usr/bin/lscpu',
|
|
'/usr/bin/gawk',
|
|
'/usr/bin/gzip',
|
|
'/usr/sbin/ifconfig',
|
|
'/usr/sbin/ethtool',
|
|
'/usr/bin/netstat',
|
|
'/usr/bin/hwloc-distrib',
|
|
'/usr/bin/hwloc-calc']
|
|
|
|
output = args.dest
|
|
|
|
libs = {}
|
|
for exe in executables:
|
|
libs.update(ldd(exe))
|
|
|
|
# manually add libthread_db for debugging thread
|
|
libs.update({'libthread_db-1.0.so': '/lib64/libthread_db-1.0.so'})
|
|
|
|
ld_so = libs['ld.so']
|
|
|
|
have_gnutls = any([lib.startswith('libgnutls.so')
|
|
for lib in libs.keys()])
|
|
|
|
# Although tarfile.open() can write directly to a compressed tar by using
|
|
# the "w|gz" mode, it does so using a slow Python implementation. It is as
|
|
# much as 3 times faster (!) to output to a pipe running the external gzip
|
|
# command. We can complete the compression even faster by using the pigz
|
|
# command - a parallel implementation of gzip utilizing all processors
|
|
# instead of just one.
|
|
gzip_process = subprocess.Popen("pigz > "+output, shell=True, stdin=subprocess.PIPE)
|
|
|
|
ar = tarfile.open(fileobj=gzip_process.stdin, mode='w|')
|
|
pathlib.Path('build/SCYLLA-RELOCATABLE-FILE').touch()
|
|
ar.add('build/SCYLLA-RELOCATABLE-FILE', arcname='SCYLLA-RELOCATABLE-FILE')
|
|
|
|
for exe in executables:
|
|
basename = os.path.basename(exe)
|
|
ar.add(exe, arcname='libexec/' + basename)
|
|
for lib, libfile in libs.items():
|
|
ar.add(libfile, arcname='libreloc/' + lib)
|
|
if have_gnutls:
|
|
gnutls_config_nolink = os.path.realpath('/etc/crypto-policies/back-ends/gnutls.config')
|
|
ar.add(gnutls_config_nolink, arcname='libreloc/gnutls.config')
|
|
ar.add('conf')
|
|
ar.add('dist')
|
|
ar.add('build/SCYLLA-RELEASE-FILE', arcname='SCYLLA-RELEASE-FILE')
|
|
ar.add('build/SCYLLA-VERSION-FILE', arcname='SCYLLA-VERSION-FILE')
|
|
ar.add('build/SCYLLA-PRODUCT-FILE', arcname='SCYLLA-PRODUCT-FILE')
|
|
ar.add('seastar/scripts')
|
|
ar.add('seastar/dpdk/usertools')
|
|
ar.add('install.sh')
|
|
# scylla_post_install.sh lives at the top level together with install.sh in the src tree, but while install.sh is
|
|
# not distributed in the .rpm and .deb packages, scylla_post_install is, so we'll add it in the package
|
|
# together with the other scripts that will end up in /usr/lib/scylla
|
|
ar.add('scylla_post_install.sh', arcname="dist/common/scripts/scylla_post_install.sh")
|
|
ar.add('scripts/relocate_python_scripts.py', arcname='relocate_python_scripts.py')
|
|
ar.add('README.md')
|
|
ar.add('README-DPDK.md')
|
|
ar.add('NOTICE.txt')
|
|
ar.add('ORIGIN')
|
|
ar.add('licenses')
|
|
ar.add('swagger-ui')
|
|
ar.add('api')
|
|
ar.add('tools')
|
|
ar.add('scylla-gdb.py')
|
|
|
|
# Complete the tar output, and wait for the gzip process to complete
|
|
ar.close()
|
|
gzip_process.communicate()
|