mirror of
https://github.com/scylladb/scylladb.git
synced 2026-04-19 16:15:07 +00:00
ba3dce3815
in Python, a raw string is created using 'r' or 'R' prefix. when
creating the regex using Python string, sometimes, we have to use
"\" to escape the parenthesis so the tools like "sed" can consider
the parenthesis as a capture group. but "\" is also used to escape
strings in Python, in order to put "\" as it is, we use "\" instead
of escaping "\" with "\\" which is obscure. when generating rules,
we use multiple-lines string and do not want to have an empty line
at the beginning of the string so added "\" continuation mark.
but we fail to escape some of the "\" in the string, and just put
"\(", despite that Python accepts it after failing to find a matched
escaped char for it, and interprets it as "\\(". this should still
be considered a misuse of oversight. with python's warning enabled,
one is able see its complaints.
in this change, we escape the "\" properly.
Signed-off-by: Kefu Chai <kefu.chai@scylladb.com>
Closes scylladb/scylladb#16179
2409 lines
111 KiB
Python
Executable File
2409 lines
111 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2015-present ScyllaDB
|
|
#
|
|
|
|
#
|
|
# SPDX-License-Identifier: AGPL-3.0-or-later
|
|
#
|
|
|
|
import argparse
|
|
import os
|
|
import platform
|
|
import re
|
|
import shlex
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import textwrap
|
|
from distutils.spawn import find_executable
|
|
|
|
outdir = 'build'
|
|
|
|
tempfile.tempdir = f"{outdir}/tmp"
|
|
|
|
configure_args = str.join(' ', [shlex.quote(x) for x in sys.argv[1:] if not x.startswith('--out=')])
|
|
|
|
# distribution "internationalization", converting package names.
|
|
# Fedora name is key, values is distro -> package name dict.
|
|
i18n_xlat = {
|
|
'boost-devel': {
|
|
'debian': 'libboost-dev',
|
|
'ubuntu': 'libboost-dev (libboost1.55-dev on 14.04)',
|
|
},
|
|
}
|
|
|
|
python3_dependencies = subprocess.run('./install-dependencies.sh --print-python3-runtime-packages', shell=True, capture_output=True, encoding='utf-8').stdout.strip()
|
|
pip_dependencies = subprocess.run('./install-dependencies.sh --print-pip-runtime-packages', shell=True, capture_output=True, encoding='utf-8').stdout.strip()
|
|
pip_symlinks = subprocess.run('./install-dependencies.sh --print-pip-symlinks', shell=True, capture_output=True, encoding='utf-8').stdout.strip()
|
|
node_exporter_filename = subprocess.run('./install-dependencies.sh --print-node-exporter-filename', shell=True, capture_output=True, encoding='utf-8').stdout.strip()
|
|
node_exporter_dirname = os.path.basename(node_exporter_filename).rstrip('.tar.gz')
|
|
|
|
|
|
def get_os_ids():
|
|
if os.environ.get('NIX_CC'):
|
|
return ['linux']
|
|
|
|
if not os.path.exists('/etc/os-release'):
|
|
return ['unknown']
|
|
|
|
os_ids = []
|
|
for line in open('/etc/os-release'):
|
|
key, _, value = line.partition('=')
|
|
value = value.strip().strip('"')
|
|
if key == 'ID':
|
|
os_ids = [value]
|
|
if key == 'ID_LIKE':
|
|
os_ids += value.split(' ')
|
|
if os_ids:
|
|
return os_ids
|
|
return ['linux'] # default ID per os-release(5)
|
|
|
|
|
|
def pkgname(name):
|
|
if name in i18n_xlat:
|
|
dict = i18n_xlat[name]
|
|
for id in get_os_ids():
|
|
if id in dict:
|
|
return dict[id]
|
|
return name
|
|
|
|
|
|
def get_flags():
|
|
with open('/proc/cpuinfo') as f:
|
|
for line in f:
|
|
if line.strip():
|
|
if line.rstrip('\n').startswith('flags'):
|
|
return re.sub(r'^flags\s+: ', '', line).split()
|
|
|
|
|
|
def add_tristate(arg_parser, name, dest, help):
|
|
arg_parser.add_argument('--enable-' + name, dest=dest, action='store_true', default=None,
|
|
help='Enable ' + help)
|
|
arg_parser.add_argument('--disable-' + name, dest=dest, action='store_false', default=None,
|
|
help='Disable ' + help)
|
|
|
|
|
|
def apply_tristate(var, test, note, missing):
|
|
if (var is None) or var:
|
|
if test():
|
|
return True
|
|
elif var is True:
|
|
print(missing)
|
|
sys.exit(1)
|
|
else:
|
|
print(note)
|
|
return False
|
|
return False
|
|
|
|
|
|
def have_pkg(package):
|
|
return subprocess.call(['pkg-config', package]) == 0
|
|
|
|
|
|
def pkg_config(package, *options):
|
|
pkg_config_path = os.environ.get('PKG_CONFIG_PATH', '')
|
|
# Add the directory containing the package to the search path, if a file is
|
|
# specified instead of a name.
|
|
if package.endswith('.pc'):
|
|
local_path = os.path.dirname(package)
|
|
pkg_config_path = '{}:{}'.format(local_path, pkg_config_path)
|
|
|
|
output = subprocess.check_output(['pkg-config'] + list(options) + [package],
|
|
env = {**os.environ,
|
|
'PKG_CONFIG_PATH': pkg_config_path})
|
|
|
|
return output.decode('utf-8').strip()
|
|
|
|
|
|
def try_compile(compiler, source='', flags=[]):
|
|
return try_compile_and_link(compiler, source, flags=flags + ['-c'])
|
|
|
|
|
|
def try_compile_and_link(compiler, source='', flags=[], verbose=False):
|
|
os.makedirs(tempfile.tempdir, exist_ok=True)
|
|
with tempfile.NamedTemporaryFile() as sfile:
|
|
ofd, ofile = tempfile.mkstemp()
|
|
os.close(ofd)
|
|
try:
|
|
sfile.file.write(bytes(source, 'utf-8'))
|
|
sfile.file.flush()
|
|
ret = subprocess.run([compiler, '-x', 'c++', '-o', ofile, sfile.name] + args.user_cflags.split() + flags,
|
|
capture_output=True)
|
|
if verbose:
|
|
print(f"Compilation failed: {compiler} -x c++ -o {ofile} {sfile.name} {args.user_cflags} {flags}")
|
|
print(source)
|
|
print(ret.stdout.decode('utf-8'))
|
|
print(ret.stderr.decode('utf-8'))
|
|
return ret.returncode == 0
|
|
finally:
|
|
if os.path.exists(ofile):
|
|
os.unlink(ofile)
|
|
|
|
|
|
def flag_supported(flag, compiler):
|
|
# gcc ignores -Wno-x even if it is not supported
|
|
adjusted = re.sub('^-Wno-', '-W', flag)
|
|
split = adjusted.split(' ')
|
|
return try_compile(flags=['-Werror'] + split, compiler=compiler)
|
|
|
|
|
|
def linker_flags(compiler):
|
|
src_main = 'int main(int argc, char **argv) { return 0; }'
|
|
link_flags = ['-fuse-ld=lld']
|
|
if try_compile_and_link(source=src_main, flags=link_flags, compiler=compiler):
|
|
print('Note: using the lld linker')
|
|
return ' '.join(link_flags)
|
|
link_flags = ['-fuse-ld=gold']
|
|
if try_compile_and_link(source=src_main, flags=link_flags, compiler=compiler):
|
|
print('Note: using the gold linker')
|
|
threads_flag = '-Wl,--threads'
|
|
if try_compile_and_link(source=src_main, flags=link_flags + [threads_flag], compiler=compiler):
|
|
link_flags.append(threads_flag)
|
|
return ' '.join(link_flags)
|
|
else:
|
|
linker = ''
|
|
try:
|
|
subprocess.call(["gold", "-v"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
linker = 'gold'
|
|
except:
|
|
pass
|
|
try:
|
|
subprocess.call(["lld", "-v"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
|
linker = 'lld'
|
|
except:
|
|
pass
|
|
if linker:
|
|
print(f'Linker {linker} found, but the compilation attempt failed, defaulting to default system linker')
|
|
else:
|
|
print('Note: neither lld nor gold found; using default system linker')
|
|
return ''
|
|
|
|
|
|
def maybe_static(flag, libs):
|
|
if flag:
|
|
libs = '-Wl,-Bstatic {} -Wl,-Bdynamic'.format(libs)
|
|
return libs
|
|
|
|
|
|
class Source(object):
|
|
def __init__(self, source, hh_prefix, cc_prefix):
|
|
self.source = source
|
|
self.hh_prefix = hh_prefix
|
|
self.cc_prefix = cc_prefix
|
|
|
|
def headers(self, gen_dir):
|
|
return [x for x in self.generated(gen_dir) if x.endswith(self.hh_prefix)]
|
|
|
|
def sources(self, gen_dir):
|
|
return [x for x in self.generated(gen_dir) if x.endswith(self.cc_prefix)]
|
|
|
|
def objects(self, gen_dir):
|
|
return [x.replace(self.cc_prefix, '.o') for x in self.sources(gen_dir)]
|
|
|
|
def endswith(self, end):
|
|
return self.source.endswith(end)
|
|
|
|
class Thrift(Source):
|
|
def __init__(self, source, service):
|
|
Source.__init__(self, source, '.h', '.cpp')
|
|
self.service = service
|
|
|
|
def generated(self, gen_dir):
|
|
basename = os.path.splitext(os.path.basename(self.source))[0]
|
|
files = [basename + '_' + ext
|
|
for ext in ['types.cpp', 'types.h', 'constants.cpp', 'constants.h']]
|
|
files += [self.service + ext
|
|
for ext in ['.cpp', '.h']]
|
|
return [os.path.join(gen_dir, file) for file in files]
|
|
|
|
def default_target_arch():
|
|
if platform.machine() in ['i386', 'i686', 'x86_64']:
|
|
return 'westmere' # support PCLMUL
|
|
elif platform.machine() == 'aarch64':
|
|
return 'armv8-a+crc+crypto'
|
|
else:
|
|
return ''
|
|
|
|
|
|
class Antlr3Grammar(Source):
|
|
def __init__(self, source):
|
|
Source.__init__(self, source, '.hpp', '.cpp')
|
|
|
|
def generated(self, gen_dir):
|
|
basename = os.path.splitext(self.source)[0]
|
|
files = [basename + ext
|
|
for ext in ['Lexer.cpp', 'Lexer.hpp', 'Parser.cpp', 'Parser.hpp']]
|
|
return [os.path.join(gen_dir, file) for file in files]
|
|
|
|
class Json2Code(Source):
|
|
def __init__(self, source):
|
|
Source.__init__(self, source, '.hh', '.cc')
|
|
|
|
def generated(self, gen_dir):
|
|
return [os.path.join(gen_dir, self.source + '.hh'), os.path.join(gen_dir, self.source + '.cc')]
|
|
|
|
def find_headers(repodir, excluded_dirs):
|
|
walker = os.walk(repodir)
|
|
|
|
_, dirs, files = next(walker)
|
|
for excl_dir in excluded_dirs:
|
|
try:
|
|
dirs.remove(excl_dir)
|
|
except ValueError:
|
|
# Ignore complaints about excl_dir not being in dirs
|
|
pass
|
|
|
|
is_hh = lambda f: f.endswith('.hh')
|
|
headers = list(filter(is_hh, files))
|
|
|
|
for dirpath, _, files in walker:
|
|
if dirpath.startswith('./'):
|
|
dirpath = dirpath[2:]
|
|
headers += [os.path.join(dirpath, hh) for hh in filter(is_hh, files)]
|
|
|
|
return sorted(headers)
|
|
|
|
|
|
def generate_compdb(compdb, ninja, buildfile, modes):
|
|
# per-mode compdbs are built by taking the relevant entries from the
|
|
# output of "ninja -t compdb" and combining them with the CMake-made
|
|
# compdbs for Seastar in the relevant mode.
|
|
#
|
|
# "ninja -t compdb" output has to be filtered because
|
|
# - it contains rules for all selected modes, and several entries for
|
|
# the same source file usually confuse indexers
|
|
# - it contains lots of irrelevant entries (for linker invocations,
|
|
# header-only compilations, etc.)
|
|
os.makedirs(tempfile.tempdir, exist_ok=True)
|
|
with tempfile.NamedTemporaryFile() as ninja_compdb:
|
|
subprocess.run([ninja, '-f', buildfile, '-t', 'compdb'], stdout=ninja_compdb.file.fileno())
|
|
ninja_compdb.file.flush()
|
|
|
|
# build mode-specific compdbs
|
|
for mode in modes:
|
|
mode_out = outdir + '/' + mode
|
|
submodule_compdbs = [mode_out + '/' + submodule + '/' + compdb for submodule in ['seastar']]
|
|
with open(mode_out + '/' + compdb, 'w+b') as combined_mode_specific_compdb:
|
|
subprocess.run(['./scripts/merge-compdb.py', 'build/' + mode,
|
|
ninja_compdb.name] + submodule_compdbs, stdout=combined_mode_specific_compdb)
|
|
|
|
# sort modes by supposed indexing speed
|
|
for mode in ['dev', 'debug', 'release', 'sanitize']:
|
|
compdb_target = outdir + '/' + mode + '/' + compdb
|
|
if os.path.exists(compdb_target):
|
|
try:
|
|
os.symlink(compdb_target, compdb)
|
|
except FileExistsError:
|
|
# if there is already a valid compile_commands.json link in the
|
|
# source root, we are done.
|
|
pass
|
|
return
|
|
|
|
|
|
def check_for_minimal_compiler_version(cxx):
|
|
compiler_test_src = '''
|
|
|
|
// clang pretends to be gcc (defined __GNUC__), so we
|
|
// must check it first
|
|
#ifdef __clang__
|
|
|
|
#if __clang_major__ < 10
|
|
#error "MAJOR"
|
|
#endif
|
|
|
|
#elif defined(__GNUC__)
|
|
|
|
#if __GNUC__ < 10
|
|
#error "MAJOR"
|
|
#elif __GNUC__ == 10
|
|
#if __GNUC_MINOR__ < 1
|
|
#error "MINOR"
|
|
#elif __GNUC_MINOR__ == 1
|
|
#if __GNUC_PATCHLEVEL__ < 1
|
|
#error "PATCHLEVEL"
|
|
#endif
|
|
#endif
|
|
#endif
|
|
|
|
#else
|
|
|
|
#error "Unrecognized compiler"
|
|
|
|
#endif
|
|
|
|
int main() { return 0; }
|
|
'''
|
|
if try_compile_and_link(compiler=cxx, source=compiler_test_src):
|
|
return
|
|
try_compile_and_link(compiler=cxx, source=compiler_test_src, verbose=True)
|
|
print('Wrong compiler version or incorrect flags. '
|
|
'Scylla needs GCC >= 10.1.1 with coroutines (-fcoroutines) or '
|
|
'clang >= 10.0.0 to compile.')
|
|
sys.exit(1)
|
|
|
|
|
|
def check_for_boost(cxx):
|
|
pkg_name = pkgname("boost-devel")
|
|
if not try_compile(compiler=cxx, source='#include <boost/version.hpp>'):
|
|
print(f'Boost not installed. Please install {pkg_name}.')
|
|
sys.exit(1)
|
|
|
|
if not try_compile(compiler=cxx, source='''\
|
|
#include <boost/version.hpp>
|
|
#if BOOST_VERSION < 105500
|
|
#error Boost version too low
|
|
#endif
|
|
'''):
|
|
print(f'Installed boost version too old. Please update {pkg_name}.')
|
|
sys.exit(1)
|
|
|
|
|
|
def check_for_lz4(cxx, cflags):
|
|
if not try_compile(cxx, source=textwrap.dedent('''\
|
|
#include <lz4.h>
|
|
|
|
void m() {
|
|
LZ4_compress_default(static_cast<const char*>(0), static_cast<char*>(0), 0, 0);
|
|
}
|
|
'''), flags=cflags.split()):
|
|
print('Installed lz4-devel is too old. Please upgrade it to r129 / v1.73 and up')
|
|
sys.exit(1)
|
|
|
|
|
|
def thrift_uses_boost_share_ptr():
|
|
# thrift version detection, see #4538
|
|
proc_res = subprocess.run(["thrift", "-version"], stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
|
|
proc_res_output = proc_res.stdout.decode("utf-8")
|
|
if proc_res.returncode != 0 and not re.search(r'^Thrift version', proc_res_output):
|
|
raise Exception("Thrift compiler must be missing: {}".format(proc_res_output))
|
|
|
|
thrift_version = proc_res_output.split(" ")[-1]
|
|
thrift_boost_versions = ["0.{}.".format(n) for n in range(1, 11)]
|
|
return any(filter(thrift_version.startswith, thrift_boost_versions))
|
|
|
|
|
|
def find_ninja():
|
|
ninja = find_executable('ninja') or find_executable('ninja-build')
|
|
if ninja:
|
|
return ninja
|
|
print('Ninja executable (ninja or ninja-build) not found on PATH\n')
|
|
sys.exit(1)
|
|
|
|
|
|
modes = {
|
|
'debug': {
|
|
'cxxflags': '-DDEBUG -DSANITIZE -DDEBUG_LSA_SANITIZER -DSCYLLA_ENABLE_ERROR_INJECTION',
|
|
'cxx_ld_flags': '',
|
|
'stack-usage-threshold': 1024*40,
|
|
# -fasan -Og breaks some coroutines on aarch64, use -O0 instead
|
|
'optimization-level': ('0' if platform.machine() == 'aarch64' else 'g'),
|
|
'per_src_extra_cxxflags': {},
|
|
'cmake_build_type': 'Debug',
|
|
'can_have_debug_info': True,
|
|
'build_seastar_shared_libs': True,
|
|
'default': True,
|
|
'description': 'a mode with no optimizations, with sanitizers, and with additional debug checks enabled, used for testing',
|
|
},
|
|
'release': {
|
|
'cxxflags': '-ffunction-sections -fdata-sections ',
|
|
'cxx_ld_flags': '-Wl,--gc-sections',
|
|
'stack-usage-threshold': 1024*13,
|
|
'optimization-level': '3',
|
|
'per_src_extra_cxxflags': {},
|
|
'cmake_build_type': 'RelWithDebInfo',
|
|
'can_have_debug_info': True,
|
|
'build_seastar_shared_libs': False,
|
|
'default': True,
|
|
'description': 'a mode with optimizations and no debug checks, used for production builds',
|
|
},
|
|
'dev': {
|
|
'cxxflags': '-DDEVEL -DSEASTAR_ENABLE_ALLOC_FAILURE_INJECTION -DSCYLLA_ENABLE_ERROR_INJECTION',
|
|
'cxx_ld_flags': '',
|
|
'stack-usage-threshold': 1024*21,
|
|
'optimization-level': '2',
|
|
'per_src_extra_cxxflags': {},
|
|
'cmake_build_type': 'Dev',
|
|
'can_have_debug_info': False,
|
|
'build_seastar_shared_libs': True,
|
|
'default': True,
|
|
'description': 'a mode with no optimizations and no debug checks, optimized for fast build times, used for development',
|
|
},
|
|
'sanitize': {
|
|
'cxxflags': '-DDEBUG -DSANITIZE -DDEBUG_LSA_SANITIZER -DSCYLLA_ENABLE_ERROR_INJECTION',
|
|
'cxx_ld_flags': '',
|
|
'stack-usage-threshold': 1024*50,
|
|
'optimization-level': 's',
|
|
'per_src_extra_cxxflags': {},
|
|
'cmake_build_type': 'Sanitize',
|
|
'can_have_debug_info': True,
|
|
'build_seastar_shared_libs': False,
|
|
'default': False,
|
|
'description': 'a mode with optimizations and sanitizers enabled, used for finding memory errors',
|
|
},
|
|
'coverage': {
|
|
'cxxflags': '-fprofile-instr-generate -fcoverage-mapping -g -gz',
|
|
'cxx_ld_flags': '-fprofile-instr-generate -fcoverage-mapping',
|
|
'stack-usage-threshold': 1024*40,
|
|
'optimization-level': 'g',
|
|
'per_src_extra_cxxflags': {},
|
|
'cmake_build_type': 'Debug',
|
|
'can_have_debug_info': True,
|
|
'build_seastar_shared_libs': False,
|
|
'default': False,
|
|
'description': 'a mode exclusively used for generating test coverage reports',
|
|
},
|
|
}
|
|
|
|
scylla_tests = set([
|
|
'test/boost/UUID_test',
|
|
'test/boost/pretty_printers_test',
|
|
'test/boost/cdc_generation_test',
|
|
'test/boost/aggregate_fcts_test',
|
|
'test/boost/allocation_strategy_test',
|
|
'test/boost/alternator_unit_test',
|
|
'test/boost/anchorless_list_test',
|
|
'test/boost/auth_passwords_test',
|
|
'test/boost/auth_resource_test',
|
|
'test/boost/auth_test',
|
|
'test/boost/batchlog_manager_test',
|
|
'test/boost/big_decimal_test',
|
|
'test/boost/broken_sstable_test',
|
|
'test/boost/bytes_ostream_test',
|
|
'test/boost/cache_algorithm_test',
|
|
'test/boost/cache_flat_mutation_reader_test',
|
|
'test/boost/cached_file_test',
|
|
'test/boost/caching_options_test',
|
|
'test/boost/canonical_mutation_test',
|
|
'test/boost/cartesian_product_test',
|
|
'test/boost/castas_fcts_test',
|
|
'test/boost/cdc_test',
|
|
'test/boost/cell_locker_test',
|
|
'test/boost/checksum_utils_test',
|
|
'test/boost/chunked_vector_test',
|
|
'test/boost/chunked_managed_vector_test',
|
|
'test/boost/clustering_ranges_walker_test',
|
|
'test/boost/column_mapping_test',
|
|
'test/boost/commitlog_test',
|
|
'test/boost/compound_test',
|
|
'test/boost/compress_test',
|
|
'test/boost/config_test',
|
|
'test/boost/continuous_data_consumer_test',
|
|
'test/boost/counter_test',
|
|
'test/boost/cql_auth_query_test',
|
|
'test/boost/cql_auth_syntax_test',
|
|
'test/boost/cql_query_test',
|
|
'test/boost/cql_query_large_test',
|
|
'test/boost/cql_query_like_test',
|
|
'test/boost/cql_query_group_test',
|
|
'test/boost/cql_functions_test',
|
|
'test/boost/crc_test',
|
|
'test/boost/data_listeners_test',
|
|
'test/boost/database_test',
|
|
'test/boost/dirty_memory_manager_test',
|
|
'test/boost/duration_test',
|
|
'test/boost/dynamic_bitset_test',
|
|
'test/boost/enum_option_test',
|
|
'test/boost/enum_set_test',
|
|
'test/boost/extensions_test',
|
|
'test/boost/error_injection_test',
|
|
'test/boost/filtering_test',
|
|
'test/boost/flat_mutation_reader_test',
|
|
'test/boost/flush_queue_test',
|
|
'test/boost/fragmented_temporary_buffer_test',
|
|
'test/boost/frozen_mutation_test',
|
|
'test/boost/gossiping_property_file_snitch_test',
|
|
'test/boost/hash_test',
|
|
'test/boost/hashers_test',
|
|
'test/boost/hint_test',
|
|
'test/boost/idl_test',
|
|
'test/boost/input_stream_test',
|
|
'test/boost/json_cql_query_test',
|
|
'test/boost/json_test',
|
|
'test/boost/keys_test',
|
|
'test/boost/large_paging_state_test',
|
|
'test/boost/recent_entries_map_test',
|
|
'test/boost/like_matcher_test',
|
|
'test/boost/limiting_data_source_test',
|
|
'test/boost/linearizing_input_stream_test',
|
|
'test/boost/loading_cache_test',
|
|
'test/boost/log_heap_test',
|
|
'test/boost/estimated_histogram_test',
|
|
'test/boost/summary_test',
|
|
'test/boost/logalloc_test',
|
|
'test/boost/logalloc_standard_allocator_segment_pool_backend_test',
|
|
'test/boost/managed_vector_test',
|
|
'test/boost/managed_bytes_test',
|
|
'test/boost/intrusive_array_test',
|
|
'test/boost/map_difference_test',
|
|
'test/boost/memtable_test',
|
|
'test/boost/multishard_mutation_query_test',
|
|
'test/boost/murmur_hash_test',
|
|
'test/boost/mutation_fragment_test',
|
|
'test/boost/mutation_query_test',
|
|
'test/boost/mutation_reader_test',
|
|
'test/boost/multishard_combining_reader_as_mutation_source_test',
|
|
'test/boost/mutation_test',
|
|
'test/boost/mutation_writer_test',
|
|
'test/boost/mvcc_test',
|
|
'test/boost/network_topology_strategy_test',
|
|
'test/boost/token_metadata_test',
|
|
'test/boost/tablets_test',
|
|
'test/boost/nonwrapping_range_test',
|
|
'test/boost/observable_test',
|
|
'test/boost/partitioner_test',
|
|
'test/boost/querier_cache_test',
|
|
'test/boost/query_processor_test',
|
|
'test/boost/range_test',
|
|
'test/boost/range_tombstone_list_test',
|
|
'test/boost/reusable_buffer_test',
|
|
'test/boost/restrictions_test',
|
|
'test/boost/repair_test',
|
|
'test/boost/role_manager_test',
|
|
'test/boost/row_cache_test',
|
|
'test/boost/rust_test',
|
|
'test/boost/schema_change_test',
|
|
'test/boost/schema_registry_test',
|
|
'test/boost/secondary_index_test',
|
|
'test/boost/tracing_test',
|
|
'test/boost/index_with_paging_test',
|
|
'test/boost/serialization_test',
|
|
'test/boost/serialized_action_test',
|
|
'test/boost/small_vector_test',
|
|
'test/boost/snitch_reset_test',
|
|
'test/boost/sstable_3_x_test',
|
|
'test/boost/sstable_datafile_test',
|
|
'test/boost/sstable_generation_test',
|
|
'test/boost/sstable_mutation_test',
|
|
'test/boost/sstable_partition_index_cache_test',
|
|
'test/boost/schema_changes_test',
|
|
'test/boost/sstable_conforms_to_mutation_source_test',
|
|
'test/boost/sstable_compaction_test',
|
|
'test/boost/sstable_resharding_test',
|
|
'test/boost/sstable_directory_test',
|
|
'test/boost/sstable_test',
|
|
'test/boost/sstable_move_test',
|
|
'test/boost/statement_restrictions_test',
|
|
'test/boost/storage_proxy_test',
|
|
'test/boost/top_k_test',
|
|
'test/boost/transport_test',
|
|
'test/boost/types_test',
|
|
'test/boost/user_function_test',
|
|
'test/boost/user_types_test',
|
|
'test/boost/utf8_test',
|
|
'test/boost/view_build_test',
|
|
'test/boost/view_complex_test',
|
|
'test/boost/view_schema_test',
|
|
'test/boost/view_schema_pkey_test',
|
|
'test/boost/view_schema_ckey_test',
|
|
'test/boost/vint_serialization_test',
|
|
'test/boost/virtual_reader_test',
|
|
'test/boost/virtual_table_mutation_source_test',
|
|
'test/boost/virtual_table_test',
|
|
'test/boost/wasm_test',
|
|
'test/boost/wasm_alloc_test',
|
|
'test/boost/bptree_test',
|
|
'test/boost/btree_test',
|
|
'test/boost/radix_tree_test',
|
|
'test/boost/double_decker_test',
|
|
'test/boost/stall_free_test',
|
|
'test/boost/sstable_set_test',
|
|
'test/boost/reader_concurrency_semaphore_test',
|
|
'test/boost/service_level_controller_test',
|
|
'test/boost/schema_loader_test',
|
|
'test/boost/lister_test',
|
|
'test/boost/group0_test',
|
|
'test/boost/exception_container_test',
|
|
'test/boost/result_utils_test',
|
|
'test/boost/rate_limiter_test',
|
|
'test/boost/per_partition_rate_limit_test',
|
|
'test/boost/expr_test',
|
|
'test/boost/exceptions_optimized_test',
|
|
'test/boost/exceptions_fallback_test',
|
|
'test/boost/s3_test',
|
|
'test/boost/locator_topology_test',
|
|
'test/boost/string_format_test',
|
|
'test/boost/tagged_integer_test',
|
|
'test/boost/group0_cmd_merge_test',
|
|
'test/manual/ec2_snitch_test',
|
|
'test/manual/enormous_table_scan_test',
|
|
'test/manual/gce_snitch_test',
|
|
'test/manual/gossip',
|
|
'test/manual/hint_test',
|
|
'test/manual/message',
|
|
'test/manual/partition_data_test',
|
|
'test/manual/row_locker_test',
|
|
'test/manual/streaming_histogram_test',
|
|
'test/manual/sstable_scan_footprint_test',
|
|
'test/perf/memory_footprint_test',
|
|
'test/perf/perf_cache_eviction',
|
|
'test/perf/perf_commitlog',
|
|
'test/perf/perf_cql_parser',
|
|
'test/perf/perf_hash',
|
|
'test/perf/perf_mutation',
|
|
'test/perf/perf_collection',
|
|
'test/perf/perf_row_cache_reads',
|
|
'test/perf/logalloc',
|
|
'test/perf/perf_s3_client',
|
|
'test/unit/lsa_async_eviction_test',
|
|
'test/unit/lsa_sync_eviction_test',
|
|
'test/unit/row_cache_alloc_stress_test',
|
|
'test/unit/row_cache_stress_test',
|
|
'test/unit/bptree_stress_test',
|
|
'test/unit/btree_stress_test',
|
|
'test/unit/bptree_compaction_test',
|
|
'test/unit/btree_compaction_test',
|
|
'test/unit/radix_tree_stress_test',
|
|
'test/unit/radix_tree_compaction_test',
|
|
'test/unit/cross_shard_barrier_test',
|
|
])
|
|
|
|
perf_tests = set([
|
|
'test/perf/perf_mutation_readers',
|
|
'test/perf/perf_checksum',
|
|
'test/perf/perf_mutation_fragment',
|
|
'test/perf/perf_idl',
|
|
'test/perf/perf_vint',
|
|
'test/perf/perf_big_decimal',
|
|
])
|
|
|
|
raft_tests = set([
|
|
'test/raft/replication_test',
|
|
'test/raft/randomized_nemesis_test',
|
|
'test/raft/many_test',
|
|
'test/raft/raft_server_test',
|
|
'test/raft/fsm_test',
|
|
'test/raft/etcd_test',
|
|
'test/raft/raft_sys_table_storage_test',
|
|
'test/raft/raft_address_map_test',
|
|
'test/raft/discovery_test',
|
|
'test/raft/failure_detector_test',
|
|
])
|
|
|
|
wasms = set([
|
|
'wasm/return_input.wat',
|
|
'wasm/test_complex_null_values.wat',
|
|
'wasm/test_fib_called_on_null.wat',
|
|
'wasm/test_functions_with_frozen_types.wat',
|
|
'wasm/test_mem_grow.wat',
|
|
'wasm/test_pow.wat',
|
|
'wasm/test_short_ints.wat',
|
|
'wasm/test_types_with_and_without_nulls.wat',
|
|
'wasm/test_UDA_final.wat',
|
|
'wasm/test_UDA_scalar.wat',
|
|
'wasm/test_word_double.wat',
|
|
])
|
|
|
|
apps = set([
|
|
'scylla',
|
|
])
|
|
|
|
tests = scylla_tests | perf_tests | raft_tests
|
|
|
|
other = set([
|
|
'iotune',
|
|
])
|
|
|
|
all_artifacts = apps | tests | other | wasms
|
|
|
|
arg_parser = argparse.ArgumentParser('Configure scylla')
|
|
arg_parser.add_argument('--out', dest='buildfile', action='store', default='build.ninja',
|
|
help='Output build-file name (by default build.ninja)')
|
|
arg_parser.add_argument('--mode', action='append', choices=list(modes.keys()), dest='selected_modes',
|
|
help="Build modes to generate ninja files for. The available build modes are:\n{}".format("; ".join(["{} - {}".format(m, cfg['description']) for m, cfg in modes.items()])))
|
|
arg_parser.add_argument('--with', dest='artifacts', action='append', default=[],
|
|
help="Specify the artifacts to build, invoke {} with --list-artifacts to list all available artifacts, if unspecified all artifacts are built".format(sys.argv[0]))
|
|
arg_parser.add_argument('--with-seastar', action='store', dest='seastar_path', default='seastar', help='Path to Seastar sources')
|
|
add_tristate(arg_parser, name='dist', dest='enable_dist',
|
|
help='scylla-tools-java, scylla-jmx and packages')
|
|
arg_parser.add_argument('--dist-only', dest='dist_only', action='store_true', default=False,
|
|
help='skip compiling code and run dist targets only')
|
|
|
|
arg_parser.add_argument('--cflags', action='store', dest='user_cflags', default='',
|
|
help='Extra flags for the C++ compiler')
|
|
arg_parser.add_argument('--ldflags', action='store', dest='user_ldflags', default='',
|
|
help='Extra flags for the linker')
|
|
arg_parser.add_argument('--target', action='store', dest='target', default=default_target_arch(),
|
|
help='Target architecture (-march)')
|
|
arg_parser.add_argument('--compiler', action='store', dest='cxx', default='clang++',
|
|
help='C++ compiler path')
|
|
arg_parser.add_argument('--c-compiler', action='store', dest='cc', default='clang',
|
|
help='C compiler path')
|
|
add_tristate(arg_parser, name='dpdk', dest='dpdk',
|
|
help='Use dpdk (from seastar dpdk sources) (default=True for release builds)')
|
|
arg_parser.add_argument('--dpdk-target', action='store', dest='dpdk_target', default='',
|
|
help='Path to DPDK SDK target location (e.g. <DPDK SDK dir>/x86_64-native-linuxapp-gcc)')
|
|
arg_parser.add_argument('--debuginfo', action='store', dest='debuginfo', type=int, default=1,
|
|
help='Enable(1)/disable(0)compiler debug information generation')
|
|
arg_parser.add_argument('--optimization-level', action='append', dest='mode_o_levels', metavar='MODE=LEVEL', default=[],
|
|
help=f'Override default compiler optimization level for mode (defaults: {" ".join([x+"="+modes[x]["optimization-level"] for x in modes])})')
|
|
arg_parser.add_argument('--static-stdc++', dest='staticcxx', action='store_true',
|
|
help='Link libgcc and libstdc++ statically')
|
|
arg_parser.add_argument('--static-thrift', dest='staticthrift', action='store_true',
|
|
help='Link libthrift statically')
|
|
arg_parser.add_argument('--static-boost', dest='staticboost', action='store_true',
|
|
help='Link boost statically')
|
|
arg_parser.add_argument('--static-yaml-cpp', dest='staticyamlcpp', action='store_true',
|
|
help='Link libyaml-cpp statically')
|
|
arg_parser.add_argument('--tests-debuginfo', action='store', dest='tests_debuginfo', type=int, default=0,
|
|
help='Enable(1)/disable(0)compiler debug information generation for tests')
|
|
arg_parser.add_argument('--perf-tests-debuginfo', action='store', dest='perf_tests_debuginfo', type=int, default=0,
|
|
help='Enable(1)/disable(0)compiler debug information generation for perf tests')
|
|
arg_parser.add_argument('--python', action='store', dest='python', default='python3',
|
|
help='Python3 path')
|
|
arg_parser.add_argument('--split-dwarf', dest='split_dwarf', action='store_true', default=False,
|
|
help='use of split dwarf (https://gcc.gnu.org/wiki/DebugFission) to speed up linking')
|
|
arg_parser.add_argument('--enable-alloc-failure-injector', dest='alloc_failure_injector', action='store_true', default=False,
|
|
help='enable allocation failure injection')
|
|
arg_parser.add_argument('--enable-seastar-debug-allocations', dest='seastar_debug_allocations', action='store_true', default=False,
|
|
help='enable seastar debug allocations')
|
|
arg_parser.add_argument('--with-antlr3', dest='antlr3_exec', action='store', default="antlr3",
|
|
help='path to antlr3 executable')
|
|
arg_parser.add_argument('--with-ragel', dest='ragel_exec', action='store', default='ragel',
|
|
help='path to ragel executable')
|
|
add_tristate(arg_parser, name='stack-guards', dest='stack_guards', help='Use stack guards')
|
|
arg_parser.add_argument('--verbose', dest='verbose', action='store_true',
|
|
help='Make configure.py output more verbose (useful for debugging the build process itself)')
|
|
arg_parser.add_argument('--test-repeat', dest='test_repeat', action='store', type=str, default='1',
|
|
help='Set number of times to repeat each unittest.')
|
|
arg_parser.add_argument('--test-timeout', dest='test_timeout', action='store', type=str, default='7200')
|
|
arg_parser.add_argument('--clang-inline-threshold', action='store', type=int, dest='clang_inline_threshold', default=-1,
|
|
help="LLVM-specific inline threshold compilation parameter")
|
|
arg_parser.add_argument('--list-artifacts', dest='list_artifacts', action='store_true', default=False,
|
|
help='List all available build artifacts, that can be passed to --with')
|
|
arg_parser.add_argument('--date-stamp', dest='date_stamp', type=str,
|
|
help='Set datestamp for SCYLLA-VERSION-GEN')
|
|
args = arg_parser.parse_args()
|
|
|
|
if args.list_artifacts:
|
|
for artifact in sorted(all_artifacts):
|
|
print(artifact)
|
|
exit(0)
|
|
|
|
defines = ['XXH_PRIVATE_API',
|
|
'SEASTAR_TESTING_MAIN',
|
|
'FMT_DEPRECATED_OSTREAM',
|
|
]
|
|
|
|
scylla_raft_core = [
|
|
'raft/raft.cc',
|
|
'raft/server.cc',
|
|
'raft/fsm.cc',
|
|
'raft/tracker.cc',
|
|
'raft/log.cc',
|
|
]
|
|
|
|
scylla_core = (['message/messaging_service.cc',
|
|
'replica/database.cc',
|
|
'replica/table.cc',
|
|
'replica/tablets.cc',
|
|
'replica/distributed_loader.cc',
|
|
'replica/memtable.cc',
|
|
'replica/exceptions.cc',
|
|
'replica/dirty_memory_manager.cc',
|
|
'replica/mutation_dump.cc',
|
|
'mutation/atomic_cell.cc',
|
|
'mutation/canonical_mutation.cc',
|
|
'mutation/frozen_mutation.cc',
|
|
'mutation/mutation.cc',
|
|
'mutation/mutation_fragment.cc',
|
|
'mutation/mutation_fragment_stream_validator.cc',
|
|
'mutation/mutation_partition.cc',
|
|
'mutation/mutation_partition_v2.cc',
|
|
'mutation/mutation_partition_view.cc',
|
|
'mutation/mutation_partition_serializer.cc',
|
|
'mutation/partition_version.cc',
|
|
'mutation/range_tombstone.cc',
|
|
'mutation/range_tombstone_list.cc',
|
|
'absl-flat_hash_map.cc',
|
|
'collection_mutation.cc',
|
|
'client_data.cc',
|
|
'debug.cc',
|
|
'schema/caching_options.cc',
|
|
'schema/schema.cc',
|
|
'schema/schema_registry.cc',
|
|
'frozen_schema.cc',
|
|
'bytes.cc',
|
|
'timeout_config.cc',
|
|
'row_cache.cc',
|
|
'schema_mutations.cc',
|
|
'generic_server.cc',
|
|
'utils/array-search.cc',
|
|
'utils/base64.cc',
|
|
'utils/logalloc.cc',
|
|
'utils/large_bitset.cc',
|
|
'utils/buffer_input_stream.cc',
|
|
'utils/limiting_data_source.cc',
|
|
'utils/updateable_value.cc',
|
|
'utils/directories.cc',
|
|
'gms/generation-number.cc',
|
|
'utils/rjson.cc',
|
|
'utils/human_readable.cc',
|
|
'utils/histogram_metrics_helper.cc',
|
|
'utils/pretty_printers.cc',
|
|
'converting_mutation_partition_applier.cc',
|
|
'readers/combined.cc',
|
|
'readers/multishard.cc',
|
|
'readers/mutation_reader.cc',
|
|
'readers/mutation_readers.cc',
|
|
'mutation_query.cc',
|
|
'keys.cc',
|
|
'counters.cc',
|
|
'compress.cc',
|
|
'zstd.cc',
|
|
'sstables/sstables.cc',
|
|
'sstables/sstables_manager.cc',
|
|
'sstables/sstable_set.cc',
|
|
'sstables/storage.cc',
|
|
'sstables/mx/partition_reversing_data_source.cc',
|
|
'sstables/mx/reader.cc',
|
|
'sstables/mx/writer.cc',
|
|
'sstables/kl/reader.cc',
|
|
'sstables/sstable_version.cc',
|
|
'sstables/compress.cc',
|
|
'sstables/sstable_mutation_reader.cc',
|
|
'compaction/compaction.cc',
|
|
'compaction/compaction_strategy.cc',
|
|
'compaction/size_tiered_compaction_strategy.cc',
|
|
'compaction/leveled_compaction_strategy.cc',
|
|
'compaction/task_manager_module.cc',
|
|
'compaction/time_window_compaction_strategy.cc',
|
|
'compaction/compaction_manager.cc',
|
|
'sstables/integrity_checked_file_impl.cc',
|
|
'sstables/prepended_input_stream.cc',
|
|
'sstables/m_format_read_helpers.cc',
|
|
'sstables/sstable_directory.cc',
|
|
'sstables/random_access_reader.cc',
|
|
'sstables/metadata_collector.cc',
|
|
'sstables/writer.cc',
|
|
'transport/cql_protocol_extension.cc',
|
|
'transport/event.cc',
|
|
'transport/event_notifier.cc',
|
|
'transport/server.cc',
|
|
'transport/controller.cc',
|
|
'transport/messages/result_message.cc',
|
|
'cdc/cdc_partitioner.cc',
|
|
'cdc/log.cc',
|
|
'cdc/split.cc',
|
|
'cdc/generation.cc',
|
|
'cdc/metadata.cc',
|
|
'cql3/type_json.cc',
|
|
'cql3/attributes.cc',
|
|
'cql3/cf_name.cc',
|
|
'cql3/cql3_type.cc',
|
|
'cql3/operation.cc',
|
|
'cql3/index_name.cc',
|
|
'cql3/keyspace_element_name.cc',
|
|
'cql3/lists.cc',
|
|
'cql3/sets.cc',
|
|
'cql3/maps.cc',
|
|
'cql3/values.cc',
|
|
'cql3/expr/expression.cc',
|
|
'cql3/expr/restrictions.cc',
|
|
'cql3/expr/prepare_expr.cc',
|
|
'cql3/functions/user_function.cc',
|
|
'cql3/functions/functions.cc',
|
|
'cql3/functions/aggregate_fcts.cc',
|
|
'cql3/functions/castas_fcts.cc',
|
|
'cql3/functions/error_injection_fcts.cc',
|
|
'cql3/statements/cf_prop_defs.cc',
|
|
'cql3/statements/cf_statement.cc',
|
|
'cql3/statements/authentication_statement.cc',
|
|
'cql3/statements/create_keyspace_statement.cc',
|
|
'cql3/statements/create_table_statement.cc',
|
|
'cql3/statements/create_view_statement.cc',
|
|
'cql3/statements/create_type_statement.cc',
|
|
'cql3/statements/create_function_statement.cc',
|
|
'cql3/statements/create_aggregate_statement.cc',
|
|
'cql3/statements/drop_index_statement.cc',
|
|
'cql3/statements/drop_keyspace_statement.cc',
|
|
'cql3/statements/drop_table_statement.cc',
|
|
'cql3/statements/drop_view_statement.cc',
|
|
'cql3/statements/drop_type_statement.cc',
|
|
'cql3/statements/drop_function_statement.cc',
|
|
'cql3/statements/drop_aggregate_statement.cc',
|
|
'cql3/statements/schema_altering_statement.cc',
|
|
'cql3/statements/ks_prop_defs.cc',
|
|
'cql3/statements/function_statement.cc',
|
|
'cql3/statements/modification_statement.cc',
|
|
'cql3/statements/cas_request.cc',
|
|
'cql3/statements/raw/parsed_statement.cc',
|
|
'cql3/statements/property_definitions.cc',
|
|
'cql3/statements/update_statement.cc',
|
|
'cql3/statements/strongly_consistent_modification_statement.cc',
|
|
'cql3/statements/strongly_consistent_select_statement.cc',
|
|
'cql3/statements/delete_statement.cc',
|
|
'cql3/statements/prune_materialized_view_statement.cc',
|
|
'cql3/statements/batch_statement.cc',
|
|
'cql3/statements/select_statement.cc',
|
|
'cql3/statements/use_statement.cc',
|
|
'cql3/statements/index_prop_defs.cc',
|
|
'cql3/statements/index_target.cc',
|
|
'cql3/statements/create_index_statement.cc',
|
|
'cql3/statements/truncate_statement.cc',
|
|
'cql3/statements/alter_table_statement.cc',
|
|
'cql3/statements/alter_view_statement.cc',
|
|
'cql3/statements/list_users_statement.cc',
|
|
'cql3/statements/authorization_statement.cc',
|
|
'cql3/statements/permission_altering_statement.cc',
|
|
'cql3/statements/list_permissions_statement.cc',
|
|
'cql3/statements/grant_statement.cc',
|
|
'cql3/statements/revoke_statement.cc',
|
|
'cql3/statements/alter_type_statement.cc',
|
|
'cql3/statements/alter_keyspace_statement.cc',
|
|
'cql3/statements/role-management-statements.cc',
|
|
'cql3/statements/service_level_statement.cc',
|
|
'cql3/statements/create_service_level_statement.cc',
|
|
'cql3/statements/alter_service_level_statement.cc',
|
|
'cql3/statements/sl_prop_defs.cc',
|
|
'cql3/statements/drop_service_level_statement.cc',
|
|
'cql3/statements/attach_service_level_statement.cc',
|
|
'cql3/statements/detach_service_level_statement.cc',
|
|
'cql3/statements/list_service_level_statement.cc',
|
|
'cql3/statements/list_service_level_attachments_statement.cc',
|
|
'cql3/statements/describe_statement.cc',
|
|
'cql3/update_parameters.cc',
|
|
'cql3/util.cc',
|
|
'cql3/ut_name.cc',
|
|
'cql3/role_name.cc',
|
|
'data_dictionary/data_dictionary.cc',
|
|
'thrift/handler.cc',
|
|
'thrift/server.cc',
|
|
'thrift/controller.cc',
|
|
'thrift/thrift_validation.cc',
|
|
'utils/runtime.cc',
|
|
'utils/murmur_hash.cc',
|
|
'utils/uuid.cc',
|
|
'utils/big_decimal.cc',
|
|
'types/types.cc',
|
|
'validation.cc',
|
|
'service/migration_manager.cc',
|
|
'service/tablet_allocator.cc',
|
|
'service/storage_proxy.cc',
|
|
'query_ranges_to_vnodes.cc',
|
|
'service/forward_service.cc',
|
|
'service/paxos/proposal.cc',
|
|
'service/paxos/prepare_response.cc',
|
|
'service/paxos/paxos_state.cc',
|
|
'service/paxos/prepare_summary.cc',
|
|
'cql3/column_identifier.cc',
|
|
'cql3/column_specification.cc',
|
|
'cql3/constants.cc',
|
|
'cql3/query_processor.cc',
|
|
'cql3/query_options.cc',
|
|
'cql3/user_types.cc',
|
|
'cql3/untyped_result_set.cc',
|
|
'cql3/selection/selectable.cc',
|
|
'cql3/selection/selection.cc',
|
|
'cql3/selection/selector.cc',
|
|
'cql3/restrictions/statement_restrictions.cc',
|
|
'cql3/result_set.cc',
|
|
'cql3/prepare_context.cc',
|
|
'db/consistency_level.cc',
|
|
'db/system_keyspace.cc',
|
|
'db/virtual_table.cc',
|
|
'db/virtual_tables.cc',
|
|
'db/system_distributed_keyspace.cc',
|
|
'db/size_estimates_virtual_reader.cc',
|
|
'db/schema_tables.cc',
|
|
'db/cql_type_parser.cc',
|
|
'db/legacy_schema_migrator.cc',
|
|
'db/commitlog/commitlog.cc',
|
|
'db/commitlog/commitlog_replayer.cc',
|
|
'db/commitlog/commitlog_entry.cc',
|
|
'db/data_listeners.cc',
|
|
'db/functions/function.cc',
|
|
'db/hints/internal/hint_endpoint_manager.cc',
|
|
'db/hints/internal/hint_sender.cc',
|
|
'db/hints/internal/hint_storage.cc',
|
|
'db/hints/manager.cc',
|
|
'db/hints/resource_manager.cc',
|
|
'db/hints/host_filter.cc',
|
|
'db/hints/sync_point.cc',
|
|
'db/config.cc',
|
|
'db/extensions.cc',
|
|
'db/heat_load_balance.cc',
|
|
'db/large_data_handler.cc',
|
|
'db/marshal/type_parser.cc',
|
|
'db/batchlog_manager.cc',
|
|
'db/tags/utils.cc',
|
|
'db/view/view.cc',
|
|
'db/view/view_update_generator.cc',
|
|
'db/view/row_locking.cc',
|
|
'db/sstables-format-selector.cc',
|
|
'db/snapshot-ctl.cc',
|
|
'db/rate_limiter.cc',
|
|
'db/per_partition_rate_limit_options.cc',
|
|
'index/secondary_index_manager.cc',
|
|
'index/secondary_index.cc',
|
|
'utils/UUID_gen.cc',
|
|
'utils/i_filter.cc',
|
|
'utils/bloom_filter.cc',
|
|
'utils/bloom_calculations.cc',
|
|
'utils/rate_limiter.cc',
|
|
'utils/file_lock.cc',
|
|
'utils/dynamic_bitset.cc',
|
|
'utils/managed_bytes.cc',
|
|
'utils/exceptions.cc',
|
|
'utils/config_file.cc',
|
|
'utils/multiprecision_int.cc',
|
|
'utils/gz/crc_combine.cc',
|
|
'utils/gz/crc_combine_table.cc',
|
|
'utils/s3/client.cc',
|
|
'gms/version_generator.cc',
|
|
'gms/versioned_value.cc',
|
|
'gms/gossiper.cc',
|
|
'gms/feature_service.cc',
|
|
'gms/gossip_digest_syn.cc',
|
|
'gms/gossip_digest_ack.cc',
|
|
'gms/gossip_digest_ack2.cc',
|
|
'gms/endpoint_state.cc',
|
|
'gms/application_state.cc',
|
|
'gms/inet_address.cc',
|
|
'dht/i_partitioner.cc',
|
|
'dht/token.cc',
|
|
'dht/murmur3_partitioner.cc',
|
|
'dht/boot_strapper.cc',
|
|
'dht/range_streamer.cc',
|
|
'unimplemented.cc',
|
|
'query.cc',
|
|
'query-result-set.cc',
|
|
'locator/abstract_replication_strategy.cc',
|
|
'locator/tablets.cc',
|
|
'locator/azure_snitch.cc',
|
|
'locator/simple_strategy.cc',
|
|
'locator/local_strategy.cc',
|
|
'locator/network_topology_strategy.cc',
|
|
'locator/everywhere_replication_strategy.cc',
|
|
'locator/token_metadata.cc',
|
|
'locator/snitch_base.cc',
|
|
'locator/simple_snitch.cc',
|
|
'locator/rack_inferring_snitch.cc',
|
|
'locator/gossiping_property_file_snitch.cc',
|
|
'locator/production_snitch_base.cc',
|
|
'locator/ec2_snitch.cc',
|
|
'locator/ec2_multi_region_snitch.cc',
|
|
'locator/gce_snitch.cc',
|
|
'locator/topology.cc',
|
|
'locator/util.cc',
|
|
'service/client_state.cc',
|
|
'service/storage_service.cc',
|
|
'service/misc_services.cc',
|
|
'service/pager/paging_state.cc',
|
|
'service/pager/query_pagers.cc',
|
|
'service/qos/qos_common.cc',
|
|
'service/qos/service_level_controller.cc',
|
|
'service/qos/standard_service_level_distributed_data_accessor.cc',
|
|
'streaming/stream_task.cc',
|
|
'streaming/stream_session.cc',
|
|
'streaming/stream_request.cc',
|
|
'streaming/stream_summary.cc',
|
|
'streaming/stream_transfer_task.cc',
|
|
'streaming/stream_receive_task.cc',
|
|
'streaming/stream_plan.cc',
|
|
'streaming/progress_info.cc',
|
|
'streaming/session_info.cc',
|
|
'streaming/stream_coordinator.cc',
|
|
'streaming/stream_manager.cc',
|
|
'streaming/stream_result_future.cc',
|
|
'streaming/stream_session_state.cc',
|
|
'streaming/consumer.cc',
|
|
'clocks-impl.cc',
|
|
'partition_slice_builder.cc',
|
|
'init.cc',
|
|
'utils/lister.cc',
|
|
'repair/repair.cc',
|
|
'repair/row_level.cc',
|
|
'exceptions/exceptions.cc',
|
|
'auth/allow_all_authenticator.cc',
|
|
'auth/allow_all_authorizer.cc',
|
|
'auth/authenticated_user.cc',
|
|
'auth/authenticator.cc',
|
|
'auth/common.cc',
|
|
'auth/default_authorizer.cc',
|
|
'auth/resource.cc',
|
|
'auth/roles-metadata.cc',
|
|
'auth/passwords.cc',
|
|
'auth/password_authenticator.cc',
|
|
'auth/permission.cc',
|
|
'auth/permissions_cache.cc',
|
|
'auth/service.cc',
|
|
'auth/standard_role_manager.cc',
|
|
'auth/transitional.cc',
|
|
'auth/role_or_anonymous.cc',
|
|
'auth/sasl_challenge.cc',
|
|
'auth/certificate_authenticator.cc',
|
|
'tracing/tracing.cc',
|
|
'tracing/trace_keyspace_helper.cc',
|
|
'tracing/trace_state.cc',
|
|
'tracing/traced_file.cc',
|
|
'table_helper.cc',
|
|
'tombstone_gc_options.cc',
|
|
'tombstone_gc.cc',
|
|
'utils/disk-error-handler.cc',
|
|
'utils/hashers.cc',
|
|
'utils/aws_sigv4.cc',
|
|
'duration.cc',
|
|
'vint-serialization.cc',
|
|
'utils/arch/powerpc/crc32-vpmsum/crc32_wrapper.cc',
|
|
'querier.cc',
|
|
'mutation_writer/multishard_writer.cc',
|
|
'multishard_mutation_query.cc',
|
|
'reader_concurrency_semaphore.cc',
|
|
'sstables_loader.cc',
|
|
'utils/utf8.cc',
|
|
'utils/ascii.cc',
|
|
'utils/like_matcher.cc',
|
|
'utils/error_injection.cc',
|
|
'utils/build_id.cc',
|
|
'mutation_writer/timestamp_based_splitting_writer.cc',
|
|
'mutation_writer/shard_based_splitting_writer.cc',
|
|
'mutation_writer/partition_based_splitting_writer.cc',
|
|
'mutation_writer/feed_writers.cc',
|
|
'lang/lua.cc',
|
|
'lang/wasm.cc',
|
|
'lang/wasm_alien_thread_runner.cc',
|
|
'lang/wasm_instance_cache.cc',
|
|
'service/raft/group0_state_machine.cc',
|
|
'service/raft/group0_state_machine_merger.cc',
|
|
'service/raft/raft_sys_table_storage.cc',
|
|
'serializer.cc',
|
|
'release.cc',
|
|
'service/raft/raft_rpc.cc',
|
|
'service/raft/raft_group_registry.cc',
|
|
'service/raft/discovery.cc',
|
|
'service/raft/raft_group0.cc',
|
|
'direct_failure_detector/failure_detector.cc',
|
|
'service/raft/raft_group0_client.cc',
|
|
'service/broadcast_tables/experimental/lang.cc',
|
|
'tasks/task_manager.cc',
|
|
'rust/wasmtime_bindings/src/lib.rs',
|
|
'utils/to_string.cc',
|
|
'service/topology_state_machine.cc',
|
|
'node_ops/node_ops_ctl.cc'
|
|
] + [Antlr3Grammar('cql3/Cql.g')] + [Thrift('interface/cassandra.thrift', 'Cassandra')] \
|
|
+ scylla_raft_core
|
|
)
|
|
|
|
api = ['api/api.cc',
|
|
Json2Code('api/api-doc/storage_service.json'),
|
|
Json2Code('api/api-doc/lsa.json'),
|
|
'api/storage_service.cc',
|
|
'api/token_metadata.cc',
|
|
Json2Code('api/api-doc/commitlog.json'),
|
|
'api/commitlog.cc',
|
|
Json2Code('api/api-doc/gossiper.json'),
|
|
'api/gossiper.cc',
|
|
Json2Code('api/api-doc/failure_detector.json'),
|
|
'api/failure_detector.cc',
|
|
Json2Code('api/api-doc/column_family.json'),
|
|
'api/column_family.cc',
|
|
'api/messaging_service.cc',
|
|
Json2Code('api/api-doc/messaging_service.json'),
|
|
Json2Code('api/api-doc/storage_proxy.json'),
|
|
'api/storage_proxy.cc',
|
|
Json2Code('api/api-doc/cache_service.json'),
|
|
'api/cache_service.cc',
|
|
Json2Code('api/api-doc/collectd.json'),
|
|
'api/collectd.cc',
|
|
Json2Code('api/api-doc/endpoint_snitch_info.json'),
|
|
'api/endpoint_snitch.cc',
|
|
Json2Code('api/api-doc/compaction_manager.json'),
|
|
'api/compaction_manager.cc',
|
|
Json2Code('api/api-doc/hinted_handoff.json'),
|
|
'api/hinted_handoff.cc',
|
|
Json2Code('api/api-doc/utils.json'),
|
|
'api/lsa.cc',
|
|
Json2Code('api/api-doc/stream_manager.json'),
|
|
'api/stream_manager.cc',
|
|
Json2Code('api/api-doc/system.json'),
|
|
'api/system.cc',
|
|
Json2Code('api/api-doc/task_manager.json'),
|
|
'api/task_manager.cc',
|
|
Json2Code('api/api-doc/task_manager_test.json'),
|
|
'api/task_manager_test.cc',
|
|
'api/config.cc',
|
|
Json2Code('api/api-doc/config.json'),
|
|
Json2Code('api/api-doc/metrics.json'),
|
|
'api/error_injection.cc',
|
|
Json2Code('api/api-doc/error_injection.json'),
|
|
'api/authorization_cache.cc',
|
|
Json2Code('api/api-doc/authorization_cache.json'),
|
|
]
|
|
|
|
alternator = [
|
|
'alternator/controller.cc',
|
|
'alternator/server.cc',
|
|
'alternator/executor.cc',
|
|
'alternator/stats.cc',
|
|
'alternator/serialization.cc',
|
|
'alternator/expressions.cc',
|
|
Antlr3Grammar('alternator/expressions.g'),
|
|
'alternator/conditions.cc',
|
|
'alternator/auth.cc',
|
|
'alternator/streams.cc',
|
|
'alternator/ttl.cc',
|
|
]
|
|
|
|
redis = [
|
|
'redis/controller.cc',
|
|
'redis/server.cc',
|
|
'redis/query_processor.cc',
|
|
'redis/protocol_parser.rl',
|
|
'redis/keyspace_utils.cc',
|
|
'redis/options.cc',
|
|
'redis/stats.cc',
|
|
'redis/mutation_utils.cc',
|
|
'redis/query_utils.cc',
|
|
'redis/abstract_command.cc',
|
|
'redis/command_factory.cc',
|
|
'redis/commands.cc',
|
|
'redis/lolwut.cc',
|
|
]
|
|
|
|
idls = ['idl/gossip_digest.idl.hh',
|
|
'idl/uuid.idl.hh',
|
|
'idl/range.idl.hh',
|
|
'idl/keys.idl.hh',
|
|
'idl/read_command.idl.hh',
|
|
'idl/token.idl.hh',
|
|
'idl/ring_position.idl.hh',
|
|
'idl/result.idl.hh',
|
|
'idl/frozen_mutation.idl.hh',
|
|
'idl/reconcilable_result.idl.hh',
|
|
'idl/streaming.idl.hh',
|
|
'idl/paging_state.idl.hh',
|
|
'idl/frozen_schema.idl.hh',
|
|
'idl/partition_checksum.idl.hh',
|
|
'idl/replay_position.idl.hh',
|
|
'idl/mutation.idl.hh',
|
|
'idl/query.idl.hh',
|
|
'idl/idl_test.idl.hh',
|
|
'idl/commitlog.idl.hh',
|
|
'idl/tracing.idl.hh',
|
|
'idl/consistency_level.idl.hh',
|
|
'idl/cache_temperature.idl.hh',
|
|
'idl/view.idl.hh',
|
|
'idl/messaging_service.idl.hh',
|
|
'idl/paxos.idl.hh',
|
|
'idl/raft.idl.hh',
|
|
'idl/raft_storage.idl.hh',
|
|
'idl/group0.idl.hh',
|
|
'idl/hinted_handoff.idl.hh',
|
|
'idl/storage_proxy.idl.hh',
|
|
'idl/group0_state_machine.idl.hh',
|
|
'idl/forward_request.idl.hh',
|
|
'idl/replica_exception.idl.hh',
|
|
'idl/per_partition_rate_limit_info.idl.hh',
|
|
'idl/position_in_partition.idl.hh',
|
|
'idl/experimental/broadcast_tables_lang.idl.hh',
|
|
'idl/storage_service.idl.hh',
|
|
'idl/join_node.idl.hh',
|
|
'idl/utils.idl.hh',
|
|
]
|
|
|
|
scylla_tests_generic_dependencies = [
|
|
'test/lib/cql_test_env.cc',
|
|
'test/lib/test_services.cc',
|
|
'test/lib/log.cc',
|
|
'test/lib/test_utils.cc',
|
|
'test/lib/tmpdir.cc',
|
|
'test/lib/sstable_run_based_compaction_strategy_for_tests.cc',
|
|
]
|
|
|
|
scylla_tests_dependencies = scylla_core + alternator + idls + scylla_tests_generic_dependencies + [
|
|
'test/lib/cql_assertions.cc',
|
|
'test/lib/result_set_assertions.cc',
|
|
'test/lib/mutation_source_test.cc',
|
|
'test/lib/sstable_utils.cc',
|
|
'test/lib/data_model.cc',
|
|
'test/lib/exception_utils.cc',
|
|
'test/lib/random_schema.cc',
|
|
'test/lib/key_utils.cc',
|
|
]
|
|
|
|
scylla_raft_dependencies = scylla_raft_core + ['utils/uuid.cc', 'utils/error_injection.cc', 'utils/exceptions.cc']
|
|
|
|
scylla_tools = ['tools/scylla-types.cc', 'tools/scylla-sstable.cc', 'tools/scylla-nodetool.cc', 'tools/schema_loader.cc', 'tools/utils.cc', 'tools/lua_sstable_consumer.cc']
|
|
scylla_perfs = ['test/perf/perf_fast_forward.cc',
|
|
'test/perf/perf_row_cache_update.cc',
|
|
'test/perf/perf_simple_query.cc',
|
|
'test/perf/perf_sstable.cc',
|
|
'test/perf/perf_tablets.cc',
|
|
'test/perf/perf.cc',
|
|
'test/lib/alternator_test_env.cc',
|
|
'test/lib/cql_test_env.cc',
|
|
'test/lib/log.cc',
|
|
'test/lib/test_services.cc',
|
|
'test/lib/test_utils.cc',
|
|
'test/lib/tmpdir.cc',
|
|
'test/lib/key_utils.cc',
|
|
'test/lib/random_schema.cc',
|
|
'test/lib/data_model.cc',
|
|
'seastar/tests/perf/linux_perf_event.cc']
|
|
|
|
deps = {
|
|
'scylla': idls + ['main.cc'] + scylla_core + api + alternator + redis + scylla_tools + scylla_perfs,
|
|
}
|
|
|
|
pure_boost_tests = set([
|
|
'test/boost/anchorless_list_test',
|
|
'test/boost/auth_passwords_test',
|
|
'test/boost/auth_resource_test',
|
|
'test/boost/big_decimal_test',
|
|
'test/boost/caching_options_test',
|
|
'test/boost/cartesian_product_test',
|
|
'test/boost/checksum_utils_test',
|
|
'test/boost/chunked_vector_test',
|
|
'test/boost/compress_test',
|
|
'test/boost/cql_auth_syntax_test',
|
|
'test/boost/crc_test',
|
|
'test/boost/duration_test',
|
|
'test/boost/dynamic_bitset_test',
|
|
'test/boost/enum_option_test',
|
|
'test/boost/enum_set_test',
|
|
'test/boost/idl_test',
|
|
'test/boost/json_test',
|
|
'test/boost/keys_test',
|
|
'test/boost/like_matcher_test',
|
|
'test/boost/linearizing_input_stream_test',
|
|
'test/boost/map_difference_test',
|
|
'test/boost/nonwrapping_range_test',
|
|
'test/boost/observable_test',
|
|
'test/boost/range_test',
|
|
'test/boost/range_tombstone_list_test',
|
|
'test/boost/serialization_test',
|
|
'test/boost/small_vector_test',
|
|
'test/boost/top_k_test',
|
|
'test/boost/vint_serialization_test',
|
|
'test/boost/bptree_test',
|
|
'test/boost/utf8_test',
|
|
'test/boost/string_format_test',
|
|
'test/manual/streaming_histogram_test',
|
|
])
|
|
|
|
tests_not_using_seastar_test_framework = set([
|
|
'test/boost/alternator_unit_test',
|
|
'test/boost/small_vector_test',
|
|
'test/manual/gossip',
|
|
'test/manual/message',
|
|
'test/perf/memory_footprint_test',
|
|
'test/perf/perf_cache_eviction',
|
|
'test/perf/perf_cql_parser',
|
|
'test/perf/perf_hash',
|
|
'test/perf/perf_mutation',
|
|
'test/perf/perf_collection',
|
|
'test/perf/logalloc',
|
|
'test/unit/lsa_async_eviction_test',
|
|
'test/unit/lsa_sync_eviction_test',
|
|
'test/unit/row_cache_alloc_stress_test',
|
|
'test/unit/bptree_stress_test',
|
|
'test/unit/btree_stress_test',
|
|
'test/unit/bptree_compaction_test',
|
|
'test/unit/btree_compaction_test',
|
|
'test/unit/radix_tree_stress_test',
|
|
'test/unit/radix_tree_compaction_test',
|
|
'test/manual/sstable_scan_footprint_test',
|
|
'test/unit/cross_shard_barrier_test',
|
|
]) | pure_boost_tests
|
|
|
|
for t in tests_not_using_seastar_test_framework:
|
|
if t not in scylla_tests:
|
|
raise Exception("Test %s not found in scylla_tests" % (t))
|
|
|
|
for t in sorted(scylla_tests):
|
|
deps[t] = [t + '.cc']
|
|
if t not in tests_not_using_seastar_test_framework:
|
|
deps[t] += scylla_tests_dependencies
|
|
else:
|
|
deps[t] += scylla_core + alternator + idls + scylla_tests_generic_dependencies
|
|
|
|
perf_tests_seastar_deps = [
|
|
'seastar/tests/perf/perf_tests.cc'
|
|
]
|
|
|
|
for t in sorted(perf_tests):
|
|
deps[t] = [t + '.cc'] + scylla_tests_dependencies + perf_tests_seastar_deps
|
|
deps[t] += ['test/perf/perf.cc', 'seastar/tests/perf/linux_perf_event.cc']
|
|
|
|
deps['test/boost/mutation_reader_test'] += ['test/lib/dummy_sharder.cc' ]
|
|
deps['test/boost/multishard_combining_reader_as_mutation_source_test'] += ['test/lib/dummy_sharder.cc' ]
|
|
|
|
deps['test/boost/bytes_ostream_test'] = [
|
|
"test/boost/bytes_ostream_test.cc",
|
|
"bytes.cc",
|
|
"utils/managed_bytes.cc",
|
|
"utils/logalloc.cc",
|
|
"utils/dynamic_bitset.cc",
|
|
"test/lib/log.cc",
|
|
]
|
|
deps['test/boost/input_stream_test'] = ['test/boost/input_stream_test.cc']
|
|
deps['test/boost/UUID_test'] = ['utils/UUID_gen.cc', 'test/boost/UUID_test.cc', 'utils/uuid.cc', 'utils/dynamic_bitset.cc', 'utils/hashers.cc']
|
|
deps['test/boost/murmur_hash_test'] = ['bytes.cc', 'utils/murmur_hash.cc', 'test/boost/murmur_hash_test.cc']
|
|
deps['test/boost/allocation_strategy_test'] = ['test/boost/allocation_strategy_test.cc', 'utils/logalloc.cc', 'utils/dynamic_bitset.cc']
|
|
deps['test/boost/log_heap_test'] = ['test/boost/log_heap_test.cc']
|
|
deps['test/boost/estimated_histogram_test'] = ['test/boost/estimated_histogram_test.cc']
|
|
deps['test/boost/summary_test'] = ['test/boost/summary_test.cc']
|
|
deps['test/boost/anchorless_list_test'] = ['test/boost/anchorless_list_test.cc']
|
|
deps['test/perf/perf_commitlog'] += ['test/perf/perf.cc', 'seastar/tests/perf/linux_perf_event.cc']
|
|
deps['test/perf/perf_row_cache_reads'] += ['test/perf/perf.cc', 'seastar/tests/perf/linux_perf_event.cc']
|
|
deps['test/boost/reusable_buffer_test'] = [
|
|
"test/boost/reusable_buffer_test.cc",
|
|
"test/lib/log.cc",
|
|
]
|
|
deps['test/boost/utf8_test'] = ['utils/utf8.cc', 'test/boost/utf8_test.cc']
|
|
deps['test/boost/small_vector_test'] = ['test/boost/small_vector_test.cc']
|
|
deps['test/boost/vint_serialization_test'] = ['test/boost/vint_serialization_test.cc', 'vint-serialization.cc', 'bytes.cc']
|
|
deps['test/boost/linearizing_input_stream_test'] = [
|
|
"test/boost/linearizing_input_stream_test.cc",
|
|
"test/lib/log.cc",
|
|
]
|
|
deps['test/boost/expr_test'] = ['test/boost/expr_test.cc', 'test/lib/expr_test_utils.cc'] + scylla_core
|
|
deps['test/boost/rate_limiter_test'] = ['test/boost/rate_limiter_test.cc', 'db/rate_limiter.cc']
|
|
deps['test/boost/exceptions_optimized_test'] = ['test/boost/exceptions_optimized_test.cc', 'utils/exceptions.cc']
|
|
deps['test/boost/exceptions_fallback_test'] = ['test/boost/exceptions_fallback_test.cc', 'utils/exceptions.cc']
|
|
|
|
deps['test/boost/duration_test'] += ['test/lib/exception_utils.cc']
|
|
deps['test/boost/schema_loader_test'] += ['tools/schema_loader.cc']
|
|
deps['test/boost/rust_test'] += ['rust/inc/src/lib.rs']
|
|
|
|
deps['test/boost/group0_cmd_merge_test'] += ['test/lib/expr_test_utils.cc']
|
|
|
|
deps['test/raft/replication_test'] = ['test/raft/replication_test.cc', 'test/raft/replication.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
|
|
deps['test/raft/raft_server_test'] = ['test/raft/raft_server_test.cc', 'test/raft/replication.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
|
|
deps['test/raft/randomized_nemesis_test'] = ['test/raft/randomized_nemesis_test.cc', 'direct_failure_detector/failure_detector.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
|
|
deps['test/raft/failure_detector_test'] = ['test/raft/failure_detector_test.cc', 'direct_failure_detector/failure_detector.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
|
|
deps['test/raft/many_test'] = ['test/raft/many_test.cc', 'test/raft/replication.cc', 'test/raft/helpers.cc'] + scylla_raft_dependencies
|
|
deps['test/raft/fsm_test'] = ['test/raft/fsm_test.cc', 'test/raft/helpers.cc', 'test/lib/log.cc'] + scylla_raft_dependencies
|
|
deps['test/raft/etcd_test'] = ['test/raft/etcd_test.cc', 'test/raft/helpers.cc', 'test/lib/log.cc'] + scylla_raft_dependencies
|
|
deps['test/raft/raft_sys_table_storage_test'] = ['test/raft/raft_sys_table_storage_test.cc'] + \
|
|
scylla_core + scylla_tests_generic_dependencies
|
|
deps['test/raft/raft_address_map_test'] = ['test/raft/raft_address_map_test.cc'] + scylla_core
|
|
deps['test/raft/discovery_test'] = ['test/raft/discovery_test.cc',
|
|
'test/raft/helpers.cc',
|
|
'test/lib/log.cc',
|
|
'service/raft/discovery.cc'] + scylla_raft_dependencies
|
|
|
|
wasm_deps = {}
|
|
|
|
wasm_deps['wasm/return_input.wat'] = 'test/resource/wasm/rust/return_input.rs'
|
|
wasm_deps['wasm/test_short_ints.wat'] = 'test/resource/wasm/rust/test_short_ints.rs'
|
|
wasm_deps['wasm/test_complex_null_values.wat'] = 'test/resource/wasm/rust/test_complex_null_values.rs'
|
|
wasm_deps['wasm/test_functions_with_frozen_types.wat'] = 'test/resource/wasm/rust/test_functions_with_frozen_types.rs'
|
|
wasm_deps['wasm/test_types_with_and_without_nulls.wat'] = 'test/resource/wasm/rust/test_types_with_and_without_nulls.rs'
|
|
|
|
wasm_deps['wasm/test_fib_called_on_null.wat'] = 'test/resource/wasm/c/test_fib_called_on_null.c'
|
|
wasm_deps['wasm/test_mem_grow.wat'] = 'test/resource/wasm/c/test_mem_grow.c'
|
|
wasm_deps['wasm/test_pow.wat'] = 'test/resource/wasm/c/test_pow.c'
|
|
wasm_deps['wasm/test_UDA_final.wat'] = 'test/resource/wasm/c/test_UDA_final.c'
|
|
wasm_deps['wasm/test_UDA_scalar.wat'] = 'test/resource/wasm/c/test_UDA_scalar.c'
|
|
wasm_deps['wasm/test_word_double.wat'] = 'test/resource/wasm/c/test_word_double.c'
|
|
|
|
|
|
def get_warning_options(cxx):
|
|
warnings = [
|
|
'-Wall',
|
|
'-Werror',
|
|
'-Wimplicit-fallthrough',
|
|
'-Wignored-qualifiers',
|
|
'-Wno-mismatched-tags', # clang-only
|
|
'-Wno-c++11-narrowing',
|
|
'-Wno-overloaded-virtual',
|
|
'-Wno-unused-command-line-argument',
|
|
'-Wno-unsupported-friend',
|
|
'-Wno-implicit-int-float-conversion',
|
|
# https://gcc.gnu.org/bugzilla/show_bug.cgi?id=77728
|
|
'-Wno-psabi',
|
|
'-Wno-narrowing',
|
|
'-Wno-enum-constexpr-conversion',
|
|
]
|
|
|
|
warnings = [w
|
|
for w in warnings
|
|
if flag_supported(flag=w, compiler=cxx)]
|
|
|
|
return ' '.join(warnings + ['-Wno-error=deprecated-declarations'])
|
|
|
|
|
|
def get_clang_inline_threshold():
|
|
if args.clang_inline_threshold != -1:
|
|
return args.clang_inline_threshold
|
|
elif platform.machine() == 'aarch64':
|
|
# we see miscompiles with 1200 and above with format("{}", uuid)
|
|
# also coroutine miscompiles with 600
|
|
return 300
|
|
else:
|
|
return 2500
|
|
|
|
for mode_level in args.mode_o_levels:
|
|
( mode, level ) = mode_level.split('=', 2)
|
|
if mode not in modes:
|
|
raise Exception(f'Mode {mode} is missing, cannot configure optimization level for it')
|
|
modes[mode]['optimization-level'] = level
|
|
|
|
linker_flags = linker_flags(compiler=args.cxx)
|
|
|
|
tests_link_rule = 'link' if args.tests_debuginfo else 'link_stripped'
|
|
perf_tests_link_rule = 'link' if args.perf_tests_debuginfo else 'link_stripped'
|
|
|
|
# Strip if debuginfo is disabled, otherwise we end up with partial
|
|
# debug info from the libraries we static link with
|
|
regular_link_rule = 'link' if args.debuginfo else 'link_stripped'
|
|
|
|
has_sanitize_address_use_after_scope = try_compile(compiler=args.cxx, flags=['-fsanitize-address-use-after-scope'], source='int f() {}')
|
|
|
|
defines = ' '.join(['-D' + d for d in defines])
|
|
|
|
globals().update(vars(args))
|
|
|
|
total_memory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
|
|
link_pool_depth = max(int(total_memory / 7e9), 1)
|
|
|
|
selected_modes = args.selected_modes or modes.keys()
|
|
default_modes = args.selected_modes or [mode for mode, mode_cfg in modes.items() if mode_cfg["default"]]
|
|
build_modes = {m: modes[m] for m in selected_modes}
|
|
|
|
if args.artifacts:
|
|
build_artifacts = set()
|
|
for artifact in args.artifacts:
|
|
if artifact in all_artifacts:
|
|
build_artifacts.add(artifact)
|
|
else:
|
|
print("Ignoring unknown build artifact: {}".format(artifact))
|
|
if not build_artifacts:
|
|
print("No artifacts to build, exiting")
|
|
exit(1)
|
|
else:
|
|
build_artifacts = all_artifacts
|
|
|
|
|
|
def generate_version(date_stamp):
|
|
date_stamp_opt = ''
|
|
if date_stamp:
|
|
date_stamp_opt = f'--date-stamp {date_stamp}'
|
|
status = subprocess.call(f"./SCYLLA-VERSION-GEN {date_stamp_opt}", shell=True)
|
|
if status != 0:
|
|
print('Version file generation failed')
|
|
sys.exit(1)
|
|
|
|
with open(f'{outdir}/SCYLLA-VERSION-FILE', 'r') as f:
|
|
scylla_version = f.read().strip().replace('-', '~')
|
|
with open(f'{outdir}/SCYLLA-RELEASE-FILE', 'r') as f:
|
|
scylla_release = f.read().strip()
|
|
with open(f'{outdir}/SCYLLA-PRODUCT-FILE', 'r') as f:
|
|
scylla_product = f.read().strip()
|
|
return scylla_product, scylla_version, scylla_release
|
|
|
|
|
|
# The relocatable package includes its own dynamic linker. We don't
|
|
# know the path it will be installed to, so for now use a very long
|
|
# path so that patchelf doesn't need to edit the program headers. The
|
|
# kernel imposes a limit of 4096 bytes including the null. The other
|
|
# constraint is that the build-id has to be in the first page, so we
|
|
# can't use all 4096 bytes for the dynamic linker.
|
|
# In here we just guess that 2000 extra / should be enough to cover
|
|
# any path we get installed to but not so large that the build-id is
|
|
# pushed to the second page.
|
|
# At the end of the build we check that the build-id is indeed in the
|
|
# first page. At install time we check that patchelf doesn't modify
|
|
# the program headers.
|
|
def dynamic_linker_option():
|
|
gcc_linker_output = subprocess.check_output(['gcc', '-###', '/dev/null', '-o', 't'], stderr=subprocess.STDOUT).decode('utf-8')
|
|
original_dynamic_linker = re.search('-dynamic-linker ([^ ]*)', gcc_linker_output).groups()[0]
|
|
|
|
employ_ld_trickery = True
|
|
# distro-specific setup
|
|
if os.environ.get('NIX_CC'):
|
|
employ_ld_trickery = False
|
|
|
|
if employ_ld_trickery:
|
|
# gdb has a SO_NAME_MAX_PATH_SIZE of 512, so limit the path size to
|
|
# that. The 512 includes the null at the end, hence the 511 bellow.
|
|
dynamic_linker = '/' * (511 - len(original_dynamic_linker)) + original_dynamic_linker
|
|
else:
|
|
dynamic_linker = original_dynamic_linker
|
|
return f'--dynamic-linker={dynamic_linker}'
|
|
|
|
forced_ldflags = '-Wl,'
|
|
|
|
# The default build-id used by lld is xxhash, which is 8 bytes long, but RPM
|
|
# requires build-ids to be at least 16 bytes long
|
|
# (https://github.com/rpm-software-management/rpm/issues/950), so let's
|
|
# explicitly ask for SHA1 build-ids.
|
|
forced_ldflags += '--build-id=sha1,'
|
|
|
|
forced_ldflags += dynamic_linker_option()
|
|
|
|
user_ldflags = forced_ldflags + ' ' + args.user_ldflags
|
|
|
|
curdir = os.getcwd()
|
|
user_cflags = args.user_cflags + f" -ffile-prefix-map={curdir}=."
|
|
|
|
if args.target != '':
|
|
user_cflags += ' -march=' + args.target
|
|
|
|
for mode in modes:
|
|
# Those flags are passed not only to Scylla objects, but also to libraries
|
|
# that we compile ourselves.
|
|
modes[mode]['lib_cflags'] = user_cflags
|
|
modes[mode]['lib_ldflags'] = user_ldflags + linker_flags
|
|
|
|
# cmake likes to separate things with semicolons
|
|
def semicolon_separated(*flags):
|
|
# original flags may be space separated, so convert to string still
|
|
# using spaces
|
|
f = ' '.join(flags)
|
|
return re.sub(' +', ';', f)
|
|
|
|
def real_relpath(path, start):
|
|
return os.path.relpath(os.path.realpath(path), os.path.realpath(start))
|
|
|
|
def configure_seastar(build_dir, mode, mode_config):
|
|
seastar_build_dir = os.path.join(build_dir, mode, 'seastar')
|
|
|
|
seastar_cmake_args = [
|
|
'-DCMAKE_BUILD_TYPE={}'.format(mode_config['cmake_build_type']),
|
|
'-DCMAKE_C_COMPILER={}'.format(args.cc),
|
|
'-DCMAKE_CXX_COMPILER={}'.format(args.cxx),
|
|
'-DCMAKE_EXPORT_NO_PACKAGE_REGISTRY=ON',
|
|
'-DSeastar_CXX_FLAGS=SHELL:{}'.format(mode_config['lib_cflags']),
|
|
'-DSeastar_LD_FLAGS={}'.format(semicolon_separated(mode_config['lib_ldflags'], mode_config['cxx_ld_flags'])),
|
|
'-DSeastar_CXX_DIALECT=gnu++20',
|
|
'-DSeastar_API_LEVEL=7',
|
|
'-DSeastar_UNUSED_RESULT_ERROR=ON',
|
|
'-DCMAKE_EXPORT_COMPILE_COMMANDS=ON',
|
|
'-DSeastar_SCHEDULING_GROUPS_COUNT=16',
|
|
'-DSeastar_IO_URING=OFF', # io_uring backend is not stable enough
|
|
]
|
|
|
|
if args.stack_guards is not None:
|
|
stack_guards = 'ON' if args.stack_guards else 'OFF'
|
|
seastar_cmake_args += ['-DSeastar_STACK_GUARDS={}'.format(stack_guards)]
|
|
|
|
dpdk = args.dpdk
|
|
if dpdk is None:
|
|
dpdk = platform.machine() == 'x86_64' and mode == 'release'
|
|
if dpdk:
|
|
seastar_cmake_args += ['-DSeastar_DPDK=ON', '-DSeastar_DPDK_MACHINE=westmere']
|
|
if args.split_dwarf:
|
|
seastar_cmake_args += ['-DSeastar_SPLIT_DWARF=ON']
|
|
if args.alloc_failure_injector:
|
|
seastar_cmake_args += ['-DSeastar_ALLOC_FAILURE_INJECTION=ON']
|
|
if args.seastar_debug_allocations:
|
|
seastar_cmake_args += ['-DSeastar_DEBUG_ALLOCATIONS=ON']
|
|
if modes[mode]['build_seastar_shared_libs']:
|
|
seastar_cmake_args += ['-DBUILD_SHARED_LIBS=ON']
|
|
|
|
seastar_cmd = ['cmake', '-G', 'Ninja', real_relpath(args.seastar_path, seastar_build_dir)] + seastar_cmake_args
|
|
cmake_dir = seastar_build_dir
|
|
if dpdk:
|
|
# need to cook first
|
|
cmake_dir = args.seastar_path # required by cooking.sh
|
|
relative_seastar_build_dir = os.path.join('..', seastar_build_dir) # relative to seastar/
|
|
seastar_cmd = ['./cooking.sh', '-i', 'dpdk', '-d', relative_seastar_build_dir, '--'] + seastar_cmd[4:]
|
|
|
|
if args.verbose:
|
|
print(" \\\n ".join(seastar_cmd))
|
|
os.makedirs(seastar_build_dir, exist_ok=True)
|
|
subprocess.check_call(seastar_cmd, shell=False, cwd=cmake_dir)
|
|
|
|
|
|
def query_seastar_flags(pc_file, use_shared_libs, link_static_cxx=False):
|
|
if use_shared_libs:
|
|
opt = '--shared'
|
|
else:
|
|
opt = '--static'
|
|
cflags = pkg_config(pc_file, '--cflags', opt)
|
|
libs = pkg_config(pc_file, '--libs', opt)
|
|
if use_shared_libs:
|
|
rpath = os.path.dirname(libs.split()[0])
|
|
libs = f"-Wl,-rpath='{rpath}' {libs}"
|
|
if link_static_cxx:
|
|
libs = libs.replace('-lstdc++ ', '')
|
|
|
|
testing_libs = pkg_config(pc_file.replace('seastar.pc', 'seastar-testing.pc'), '--libs', '--static')
|
|
return {'seastar_cflags': cflags,
|
|
'seastar_libs': libs,
|
|
'seastar_testing_libs': testing_libs}
|
|
|
|
pkgs = ['libsystemd',
|
|
'jsoncpp',
|
|
'absl_raw_hash_set',
|
|
'absl_hash']
|
|
# Lua can be provided by lua53 package on Debian-like
|
|
# systems and by Lua on others.
|
|
pkgs.append('lua53' if have_pkg('lua53') else 'lua')
|
|
|
|
|
|
libs = ' '.join([maybe_static(args.staticyamlcpp, '-lyaml-cpp'), '-latomic', '-llz4', '-lz', '-lsnappy',
|
|
' -lstdc++fs', ' -lcrypt', ' -lcryptopp', ' -lpthread',
|
|
# Must link with static version of libzstd, since
|
|
# experimental APIs that we use are only present there.
|
|
maybe_static(True, '-lzstd'),
|
|
maybe_static(args.staticboost, '-lboost_date_time -lboost_regex -licuuc -licui18n'),
|
|
'-lxxhash',
|
|
'-ldeflate',
|
|
])
|
|
|
|
if not args.staticboost:
|
|
user_cflags += ' -DBOOST_TEST_DYN_LINK'
|
|
|
|
if thrift_uses_boost_share_ptr():
|
|
user_cflags += ' -DTHRIFT_USES_BOOST'
|
|
|
|
for pkg in pkgs:
|
|
user_cflags += ' ' + pkg_config(pkg, '--cflags')
|
|
libs += ' ' + pkg_config(pkg, '--libs')
|
|
user_cflags += ' -fvisibility=hidden'
|
|
user_ldflags += ' -fvisibility=hidden'
|
|
if args.staticcxx:
|
|
user_ldflags += " -static-libstdc++"
|
|
|
|
|
|
def get_extra_cxxflags(mode, mode_config, cxx, debuginfo):
|
|
cxxflags = []
|
|
|
|
optimization_level = mode_config['optimization-level']
|
|
cxxflags.append(f'-O{optimization_level}')
|
|
|
|
if mode == 'release':
|
|
optimization_flags = [
|
|
'--param inline-unit-growth=300', # gcc
|
|
f'-mllvm -inline-threshold={get_clang_inline_threshold()}', # clang
|
|
# clang generates 16-byte loads that break store-to-load forwarding
|
|
# gcc also has some trouble: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=103554
|
|
'-fno-slp-vectorize',
|
|
]
|
|
optimization_flags = [o
|
|
for o in optimization_flags
|
|
if flag_supported(flag=o, compiler=cxx)]
|
|
cxxflags += optimization_flags
|
|
|
|
if flag_supported(flag='-Wstack-usage=4096', compiler=cxx):
|
|
stack_usage_threshold = mode_config['stack-usage-threshold']
|
|
cxxflags += [f'-Wstack-usage={stack_usage_threshold}',
|
|
'-Wno-error=stack-usage=']
|
|
|
|
cxxflags.append(f'-DSCYLLA_BUILD_MODE={mode}')
|
|
|
|
if debuginfo and mode_config['can_have_debug_info']:
|
|
cxxflags += ['-g', '-gz']
|
|
|
|
return cxxflags
|
|
|
|
|
|
def get_release_cxxflags(scylla_version,
|
|
scylla_release):
|
|
definitions = {'SCYLLA_VERSION': scylla_version,
|
|
'SCYLLA_RELEASE': scylla_release}
|
|
return [f'-D{name}="\\"{value}\\""' for name, value in definitions.items()]
|
|
|
|
|
|
def write_build_file(f,
|
|
arch,
|
|
ninja,
|
|
scylla_product,
|
|
scylla_version,
|
|
scylla_release,
|
|
args):
|
|
warnings = get_warning_options(args.cxx)
|
|
f.write(textwrap.dedent('''\
|
|
configure_args = {configure_args}
|
|
builddir = {outdir}
|
|
cxx = {cxx}
|
|
cxxflags = --std=gnu++20 {user_cflags} {warnings} {defines}
|
|
ldflags = {linker_flags} {user_ldflags}
|
|
ldflags_build = {linker_flags}
|
|
libs = {libs}
|
|
pool link_pool
|
|
depth = {link_pool_depth}
|
|
pool submodule_pool
|
|
depth = 1
|
|
rule gen
|
|
command = echo -e $text > $out
|
|
description = GEN $out
|
|
rule swagger
|
|
command = {seastar_path}/scripts/seastar-json2code.py --create-cc -f $in -o $out
|
|
description = SWAGGER $out
|
|
rule serializer
|
|
command = {python} ./idl-compiler.py --ns ser -f $in -o $out
|
|
description = IDL compiler $out
|
|
rule ninja
|
|
command = {ninja} -C $subdir $target
|
|
restat = 1
|
|
description = NINJA $out
|
|
rule ragel
|
|
# sed away a bug in ragel 7 that emits some extraneous _nfa* variables
|
|
# (the $$ is collapsed to a single one by ninja)
|
|
command = {ragel_exec} -G2 -o $out $in && sed -i -e '1h;2,$$H;$$!d;g' -re 's/static const char _nfa[^;]*;//g' $out
|
|
description = RAGEL $out
|
|
rule run
|
|
command = $in > $out
|
|
description = GEN $out
|
|
rule copy
|
|
command = cp --reflink=auto $in $out
|
|
description = COPY $out
|
|
rule strip
|
|
command = scripts/strip.sh $in
|
|
rule package
|
|
command = scripts/create-relocatable-package.py --build-dir build/$mode $out
|
|
rule stripped_package
|
|
command = scripts/create-relocatable-package.py --stripped --build-dir build/$mode $out
|
|
rule debuginfo_package
|
|
command = dist/debuginfo/scripts/create-relocatable-package.py --build-dir build/$mode $out
|
|
rule rpmbuild
|
|
command = reloc/build_rpm.sh --reloc-pkg $in --builddir $out
|
|
rule debbuild
|
|
command = reloc/build_deb.sh --reloc-pkg $in --builddir $out
|
|
rule unified
|
|
command = unified/build_unified.sh --build-dir build/$mode --unified-pkg $out
|
|
rule rust_header
|
|
command = cxxbridge --include rust/cxx.h --header $in > $out
|
|
description = RUST_HEADER $out
|
|
rule rust_source
|
|
command = cxxbridge --include rust/cxx.h $in > $out
|
|
description = RUST_SOURCE $out
|
|
rule cxxbridge_header
|
|
command = cxxbridge --header > $out
|
|
rule c2wasm
|
|
command = clang --target=wasm32 --no-standard-libraries -Wl,--export-all -Wl,--no-entry $in -o $out
|
|
description = C2WASM $out
|
|
rule rust2wasm
|
|
command = cargo build --target=wasm32-wasi --example=$example --locked --manifest-path=test/resource/wasm/rust/Cargo.toml --target-dir=$builddir/wasm/ $
|
|
&& wasm-opt -Oz $builddir/wasm/wasm32-wasi/debug/examples/$example.wasm -o $builddir/wasm/$example.wasm $
|
|
&& wasm-strip $builddir/wasm/$example.wasm
|
|
description = RUST2WASM $out
|
|
rule wasm2wat
|
|
command = wasm2wat $in > $out
|
|
description = WASM2WAT $out
|
|
''').format(configure_args=configure_args,
|
|
outdir=outdir,
|
|
cxx=args.cxx,
|
|
user_cflags=user_cflags,
|
|
warnings=warnings,
|
|
defines=defines,
|
|
linker_flags=linker_flags,
|
|
user_ldflags=user_ldflags,
|
|
libs=libs,
|
|
link_pool_depth=link_pool_depth,
|
|
python=args.python,
|
|
seastar_path=args.seastar_path,
|
|
ninja=ninja,
|
|
ragel_exec=args.ragel_exec))
|
|
|
|
for binary in sorted(wasms):
|
|
src = wasm_deps[binary]
|
|
wasm = binary[:-4] + '.wasm'
|
|
if src.endswith('.rs'):
|
|
f.write(f'build $builddir/{wasm}: rust2wasm {src} | test/resource/wasm/rust/Cargo.lock test/resource/wasm/rust/build.rs\n')
|
|
example_name = binary[binary.rindex('/')+1:-4]
|
|
f.write(f' example = {example_name}\n')
|
|
else:
|
|
f.write(f'build $builddir/{wasm}: c2wasm {src}\n')
|
|
f.write(f'build $builddir/{binary}: wasm2wat $builddir/{wasm}\n')
|
|
|
|
for mode in build_modes:
|
|
modeval = modes[mode]
|
|
|
|
modeval.update(query_seastar_flags(f'{outdir}/{mode}/seastar/seastar.pc',
|
|
modeval['build_seastar_shared_libs'],
|
|
args.staticcxx))
|
|
|
|
extra_cxxflags = ' '.join(get_extra_cxxflags(mode, modeval, args.cxx, args.debuginfo))
|
|
modeval['cxxflags'] += f' {extra_cxxflags}'
|
|
|
|
modeval['per_src_extra_cxxflags']['release.cc'] = ' '.join(get_release_cxxflags(scylla_version,
|
|
scylla_release))
|
|
|
|
fmt_lib = 'fmt'
|
|
f.write(textwrap.dedent('''\
|
|
cxx_ld_flags_{mode} = {cxx_ld_flags}
|
|
ld_flags_{mode} = $cxx_ld_flags_{mode} {lib_ldflags}
|
|
cxxflags_{mode} = $cxx_ld_flags_{mode} {lib_cflags} {cxxflags} -iquote. -iquote $builddir/{mode}/gen
|
|
libs_{mode} = -l{fmt_lib}
|
|
seastar_libs_{mode} = {seastar_libs}
|
|
seastar_testing_libs_{mode} = {seastar_testing_libs}
|
|
rule cxx.{mode}
|
|
command = $cxx -MD -MT $out -MF $out.d {seastar_cflags} $cxxflags_{mode} $cxxflags $obj_cxxflags -c -o $out $in
|
|
description = CXX $out
|
|
depfile = $out.d
|
|
rule link.{mode}
|
|
command = $cxx $ld_flags_{mode} $ldflags -o $out $in $libs $libs_{mode}
|
|
description = LINK $out
|
|
pool = link_pool
|
|
rule link_stripped.{mode}
|
|
command = $cxx $ld_flags_{mode} -s $ldflags -o $out $in $libs $libs_{mode}
|
|
description = LINK (stripped) $out
|
|
pool = link_pool
|
|
rule link_build.{mode}
|
|
command = $cxx $ld_flags_{mode} $ldflags_build -o $out $in $libs $libs_{mode}
|
|
description = LINK (build) $out
|
|
pool = link_pool
|
|
rule ar.{mode}
|
|
command = rm -f $out; ar cr $out $in; ranlib $out
|
|
description = AR $out
|
|
rule thrift.{mode}
|
|
command = thrift -gen cpp:cob_style -out $builddir/{mode}/gen $in
|
|
description = THRIFT $in
|
|
restat = 1
|
|
rule antlr3.{mode}
|
|
# We replace many local `ExceptionBaseType* ex` variables with a single function-scope one.
|
|
# Because we add such a variable to every function, and because `ExceptionBaseType` is not a global
|
|
# name, we also add a global typedef to avoid compilation errors.
|
|
command = sed -e '/^#if 0/,/^#endif/d' $in > $builddir/{mode}/gen/$in $
|
|
&& {antlr3_exec} $builddir/{mode}/gen/$in $
|
|
&& sed -i -e '/^.*On :.*$$/d' $builddir/{mode}/gen/${{stem}}Lexer.hpp $
|
|
&& sed -i -e '/^.*On :.*$$/d' $builddir/{mode}/gen/${{stem}}Lexer.cpp $
|
|
&& sed -i -e '/^.*On :.*$$/d' $builddir/{mode}/gen/${{stem}}Parser.hpp $
|
|
&& sed -i -e 's/^\\( *\\)\\(ImplTraits::CommonTokenType\\* [a-zA-Z0-9_]* = NULL;\\)$$/\\1const \\2/' $
|
|
-e '/^.*On :.*$$/d' $
|
|
-e '1i using ExceptionBaseType = int;' $
|
|
-e 's/^{{/{{ ExceptionBaseType\\* ex = nullptr;/; $
|
|
s/ExceptionBaseType\\* ex = new/ex = new/; $
|
|
s/exceptions::syntax_exception e/exceptions::syntax_exception\\& e/' $
|
|
$builddir/{mode}/gen/${{stem}}Parser.cpp
|
|
description = ANTLR3 $in
|
|
rule checkhh.{mode}
|
|
command = $cxx -MD -MT $out -MF $out.d {seastar_cflags} $cxxflags $cxxflags_{mode} $obj_cxxflags --include $in -c -o $out $builddir/{mode}/gen/empty.cc
|
|
description = CHECKHH $in
|
|
depfile = $out.d
|
|
rule test.{mode}
|
|
command = ./test.py --mode={mode} --repeat={test_repeat} --timeout={test_timeout}
|
|
pool = console
|
|
description = TEST {mode}
|
|
rule rust_lib.{mode}
|
|
command = CARGO_BUILD_DEP_INFO_BASEDIR='.' cargo build --locked --manifest-path=rust/Cargo.toml --target-dir=$builddir/{mode} --profile=rust-{mode} $
|
|
&& touch $out
|
|
description = RUST_LIB $out
|
|
''').format(mode=mode, antlr3_exec=args.antlr3_exec, fmt_lib=fmt_lib, test_repeat=args.test_repeat, test_timeout=args.test_timeout, **modeval))
|
|
f.write(
|
|
'build {mode}-build: phony {artifacts} {wasms}\n'.format(
|
|
mode=mode,
|
|
artifacts=str.join(' ', ['$builddir/' + mode + '/' + x for x in sorted(build_artifacts - wasms)]),
|
|
wasms = str.join(' ', ['$builddir/' + x for x in sorted(build_artifacts & wasms)]),
|
|
)
|
|
)
|
|
include_cxx_target = f'{mode}-build' if not args.dist_only else ''
|
|
include_dist_target = f'dist-{mode}' if args.enable_dist is None or args.enable_dist else ''
|
|
f.write(f'build {mode}: phony {include_cxx_target} {include_dist_target}\n')
|
|
compiles = {}
|
|
swaggers = set()
|
|
serializers = {}
|
|
thrifts = set()
|
|
ragels = {}
|
|
antlr3_grammars = set()
|
|
rust_headers = {}
|
|
seastar_lib_ext = 'so' if modeval['build_seastar_shared_libs'] else 'a'
|
|
seastar_dep = f'$builddir/{mode}/seastar/libseastar.{seastar_lib_ext}'
|
|
seastar_testing_dep = f'$builddir/{mode}/seastar/libseastar_testing.{seastar_lib_ext}'
|
|
for binary in sorted(build_artifacts):
|
|
if binary in other or binary in wasms:
|
|
continue
|
|
srcs = deps[binary]
|
|
objs = ['$builddir/' + mode + '/' + src.replace('.cc', '.o')
|
|
for src in srcs
|
|
if src.endswith('.cc')]
|
|
objs.append('$builddir/../utils/arch/powerpc/crc32-vpmsum/crc32.S')
|
|
has_thrift = False
|
|
has_rust = False
|
|
for dep in deps[binary]:
|
|
if isinstance(dep, Thrift):
|
|
has_thrift = True
|
|
objs += dep.objects('$builddir/' + mode + '/gen')
|
|
if isinstance(dep, Antlr3Grammar):
|
|
objs += dep.objects('$builddir/' + mode + '/gen')
|
|
if isinstance(dep, Json2Code):
|
|
objs += dep.objects('$builddir/' + mode + '/gen')
|
|
if dep.endswith('.rs'):
|
|
has_rust = True
|
|
idx = dep.rindex('/src/')
|
|
obj = dep[:idx].replace('rust/','') + '.o'
|
|
objs.append('$builddir/' + mode + '/gen/rust/' + obj)
|
|
if has_rust:
|
|
objs.append('$builddir/' + mode +'/rust-' + mode + '/librust_combined.a')
|
|
local_libs = '$seastar_libs_{} $libs'.format(mode)
|
|
if has_thrift:
|
|
local_libs += ' ' + maybe_static(args.staticthrift, '-lthrift')
|
|
local_libs += ' ' + maybe_static(args.staticboost, '-lboost_system')
|
|
if binary in tests:
|
|
if binary in pure_boost_tests:
|
|
local_libs += ' ' + maybe_static(args.staticboost, '-lboost_unit_test_framework')
|
|
if binary not in tests_not_using_seastar_test_framework:
|
|
local_libs += ' ' + "$seastar_testing_libs_{}".format(mode)
|
|
else:
|
|
local_libs += ' ' + '-lgnutls' + ' ' + '-lboost_unit_test_framework'
|
|
# Our code's debugging information is huge, and multiplied
|
|
# by many tests yields ridiculous amounts of disk space.
|
|
# So we strip the tests by default; The user can very
|
|
# quickly re-link the test unstripped by adding a "_g"
|
|
# to the test name, e.g., "ninja build/release/testname_g"
|
|
link_rule = perf_tests_link_rule if binary.startswith('test/perf/') else tests_link_rule
|
|
f.write('build $builddir/{}/{}: {}.{} {} | {} {}\n'.format(mode, binary, link_rule, mode, str.join(' ', objs), seastar_dep, seastar_testing_dep))
|
|
f.write(' libs = {}\n'.format(local_libs))
|
|
f.write('build $builddir/{}/{}_g: {}.{} {} | {} {}\n'.format(mode, binary, regular_link_rule, mode, str.join(' ', objs), seastar_dep, seastar_testing_dep))
|
|
f.write(' libs = {}\n'.format(local_libs))
|
|
else:
|
|
if binary == 'scylla':
|
|
local_libs += ' ' + "$seastar_testing_libs_{}".format(mode)
|
|
f.write('build $builddir/{}/{}: {}.{} {} | {} {}\n'.format(mode, binary, regular_link_rule, mode, str.join(' ', objs), seastar_dep, seastar_testing_dep))
|
|
f.write(' libs = {}\n'.format(local_libs))
|
|
f.write(f'build $builddir/{mode}/{binary}.stripped: strip $builddir/{mode}/{binary}\n')
|
|
f.write(f'build $builddir/{mode}/{binary}.debug: phony $builddir/{mode}/{binary}.stripped\n')
|
|
for src in srcs:
|
|
if src.endswith('.cc'):
|
|
obj = '$builddir/' + mode + '/' + src.replace('.cc', '.o')
|
|
compiles[obj] = src
|
|
elif src.endswith('.idl.hh'):
|
|
hh = '$builddir/' + mode + '/gen/' + src.replace('.idl.hh', '.dist.hh')
|
|
serializers[hh] = src
|
|
elif src.endswith('.json'):
|
|
swaggers.add(src)
|
|
elif src.endswith('.rl'):
|
|
hh = '$builddir/' + mode + '/gen/' + src.replace('.rl', '.hh')
|
|
ragels[hh] = src
|
|
elif src.endswith('.thrift'):
|
|
thrifts.add(src)
|
|
elif src.endswith('.g'):
|
|
antlr3_grammars.add(src)
|
|
elif src.endswith('.rs'):
|
|
idx = src.rindex('/src/')
|
|
hh = '$builddir/' + mode + '/gen/' + src[:idx] + '.hh'
|
|
rust_headers[hh] = src
|
|
else:
|
|
raise Exception('No rule for ' + src)
|
|
f.write(
|
|
'build {mode}-objects: phony {objs}\n'.format(
|
|
mode=mode,
|
|
objs=' '.join(compiles)
|
|
)
|
|
)
|
|
|
|
headers = find_headers('.', excluded_dirs=['idl', 'build', 'seastar', '.git'])
|
|
f.write(
|
|
'build {mode}-headers: phony {header_objs}\n'.format(
|
|
mode=mode,
|
|
header_objs=' '.join(["$builddir/{mode}/{hh}.o".format(mode=mode, hh=hh) for hh in headers])
|
|
)
|
|
)
|
|
|
|
f.write(
|
|
'build {mode}-test: test.{mode} {test_executables} $builddir/{mode}/scylla {wasms}\n'.format(
|
|
mode=mode,
|
|
test_executables=' '.join(['$builddir/{}/{}'.format(mode, binary) for binary in sorted(tests)]),
|
|
wasms=' '.join([f'$builddir/{binary}' for binary in sorted(wasms)]),
|
|
)
|
|
)
|
|
f.write(
|
|
'build {mode}-check: phony {mode}-headers {mode}-test\n'.format(
|
|
mode=mode,
|
|
)
|
|
)
|
|
|
|
gen_dir = '$builddir/{}/gen'.format(mode)
|
|
gen_headers = []
|
|
for th in thrifts:
|
|
gen_headers += th.headers('$builddir/{}/gen'.format(mode))
|
|
for g in antlr3_grammars:
|
|
gen_headers += g.headers('$builddir/{}/gen'.format(mode))
|
|
for g in swaggers:
|
|
gen_headers += g.headers('$builddir/{}/gen'.format(mode))
|
|
gen_headers += list(serializers.keys())
|
|
gen_headers += list(ragels.keys())
|
|
gen_headers += list(rust_headers.keys())
|
|
gen_headers.append('$builddir/{}/gen/rust/cxx.h'.format(mode))
|
|
gen_headers_dep = ' '.join(gen_headers)
|
|
|
|
for obj in compiles:
|
|
src = compiles[obj]
|
|
f.write('build {}: cxx.{} {} || {} {}\n'.format(obj, mode, src, seastar_dep, gen_headers_dep))
|
|
if src in modeval['per_src_extra_cxxflags']:
|
|
f.write(' cxxflags = {seastar_cflags} $cxxflags $cxxflags_{mode} {extra_cxxflags}\n'.format(mode=mode, extra_cxxflags=modeval["per_src_extra_cxxflags"][src], **modeval))
|
|
for swagger in swaggers:
|
|
hh = swagger.headers(gen_dir)[0]
|
|
cc = swagger.sources(gen_dir)[0]
|
|
obj = swagger.objects(gen_dir)[0]
|
|
src = swagger.source
|
|
f.write('build {} | {} : swagger {} | {}/scripts/seastar-json2code.py\n'.format(hh, cc, src, args.seastar_path))
|
|
f.write('build {}: cxx.{} {}\n'.format(obj, mode, cc))
|
|
for hh in serializers:
|
|
src = serializers[hh]
|
|
f.write('build {}: serializer {} | idl-compiler.py\n'.format(hh, src))
|
|
for hh in ragels:
|
|
src = ragels[hh]
|
|
f.write('build {}: ragel {}\n'.format(hh, src))
|
|
for hh in rust_headers:
|
|
src = rust_headers[hh]
|
|
f.write('build {}: rust_header {}\n'.format(hh, src))
|
|
cc = hh.replace('.hh', '.cc')
|
|
f.write('build {}: rust_source {}\n'.format(cc, src))
|
|
obj = cc.replace('.cc', '.o')
|
|
f.write('build {}: cxx.{} {} || {}\n'.format(obj, mode, cc, gen_headers_dep))
|
|
f.write('build {}: cxxbridge_header\n'.format('$builddir/{}/gen/rust/cxx.h'.format(mode)))
|
|
librust = '$builddir/{}/rust-{}/librust_combined'.format(mode, mode)
|
|
f.write('build {}.a: rust_lib.{} rust/Cargo.lock\n depfile={}.d\n'.format(librust, mode, librust))
|
|
for thrift in thrifts:
|
|
outs = ' '.join(thrift.generated('$builddir/{}/gen'.format(mode)))
|
|
f.write('build {}: thrift.{} {}\n'.format(outs, mode, thrift.source))
|
|
for cc in thrift.sources('$builddir/{}/gen'.format(mode)):
|
|
obj = cc.replace('.cpp', '.o')
|
|
f.write('build {}: cxx.{} {}\n'.format(obj, mode, cc))
|
|
for grammar in antlr3_grammars:
|
|
outs = ' '.join(grammar.generated('$builddir/{}/gen'.format(mode)))
|
|
f.write('build {}: antlr3.{} {}\n stem = {}\n'.format(outs, mode, grammar.source,
|
|
grammar.source.rsplit('.', 1)[0]))
|
|
for cc in grammar.sources('$builddir/{}/gen'.format(mode)):
|
|
obj = cc.replace('.cpp', '.o')
|
|
f.write('build {}: cxx.{} {} || {}\n'.format(obj, mode, cc, ' '.join(serializers)))
|
|
flags = '-Wno-parentheses-equality'
|
|
if cc.endswith('Parser.cpp'):
|
|
# Unoptimized parsers end up using huge amounts of stack space and overflowing their stack
|
|
flags += ' -O1' if modes[mode]['optimization-level'] in ['0', 'g', 's'] else ''
|
|
|
|
if has_sanitize_address_use_after_scope:
|
|
flags += ' -fno-sanitize-address-use-after-scope'
|
|
f.write(f' obj_cxxflags = {flags}\n')
|
|
f.write(f'build $builddir/{mode}/gen/empty.cc: gen\n')
|
|
for hh in headers:
|
|
f.write('build $builddir/{mode}/{hh}.o: checkhh.{mode} {hh} | $builddir/{mode}/gen/empty.cc || {gen_headers_dep}\n'.format(
|
|
mode=mode, hh=hh, gen_headers_dep=gen_headers_dep))
|
|
|
|
f.write('build {seastar_dep}: ninja $builddir/{mode}/seastar/build.ninja | always\n'
|
|
.format(**locals()))
|
|
f.write(' pool = submodule_pool\n')
|
|
f.write(' subdir = $builddir/{mode}/seastar\n'.format(**locals()))
|
|
f.write(' target = seastar\n'.format(**locals()))
|
|
f.write('build {seastar_testing_dep}: ninja $builddir/{mode}/seastar/build.ninja | always\n'
|
|
.format(**locals()))
|
|
f.write(' pool = submodule_pool\n')
|
|
f.write(' subdir = $builddir/{mode}/seastar\n'.format(**locals()))
|
|
f.write(' target = seastar_testing\n'.format(**locals()))
|
|
f.write('build $builddir/{mode}/seastar/apps/iotune/iotune: ninja $builddir/{mode}/seastar/build.ninja\n'
|
|
.format(**locals()))
|
|
f.write(' pool = submodule_pool\n')
|
|
f.write(' subdir = $builddir/{mode}/seastar\n'.format(**locals()))
|
|
f.write(' target = iotune\n'.format(**locals()))
|
|
f.write(textwrap.dedent('''\
|
|
build $builddir/{mode}/iotune: copy $builddir/{mode}/seastar/apps/iotune/iotune
|
|
build $builddir/{mode}/iotune.stripped: strip $builddir/{mode}/iotune
|
|
build $builddir/{mode}/iotune.debug: phony $builddir/{mode}/iotune.stripped
|
|
''').format(**locals()))
|
|
if args.dist_only:
|
|
include_scylla_and_iotune = ''
|
|
include_scylla_and_iotune_stripped = ''
|
|
include_scylla_and_iotune_debug = ''
|
|
else:
|
|
include_scylla_and_iotune = f'$builddir/{mode}/scylla $builddir/{mode}/iotune'
|
|
include_scylla_and_iotune_stripped = f'$builddir/{mode}/scylla.stripped $builddir/{mode}/iotune.stripped'
|
|
include_scylla_and_iotune_debug = f'$builddir/{mode}/scylla.debug $builddir/{mode}/iotune.debug'
|
|
f.write('build $builddir/{mode}/dist/tar/{scylla_product}-unstripped-{scylla_version}-{scylla_release}.{arch}.tar.gz: package {include_scylla_and_iotune} $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE $builddir/debian/debian $builddir/node_exporter/node_exporter | always\n'.format(**locals()))
|
|
f.write(' mode = {mode}\n'.format(**locals()))
|
|
f.write('build $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz: stripped_package {include_scylla_and_iotune_stripped} $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE $builddir/debian/debian $builddir/node_exporter/node_exporter.stripped | always\n'.format(**locals()))
|
|
f.write(' mode = {mode}\n'.format(**locals()))
|
|
f.write('build $builddir/{mode}/dist/tar/{scylla_product}-debuginfo-{scylla_version}-{scylla_release}.{arch}.tar.gz: debuginfo_package {include_scylla_and_iotune_debug} $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE $builddir/debian/debian $builddir/node_exporter/node_exporter.debug | always\n'.format(**locals()))
|
|
f.write(' mode = {mode}\n'.format(**locals()))
|
|
f.write('build $builddir/{mode}/dist/tar/{scylla_product}-package.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz\n'.format(**locals()))
|
|
f.write(' mode = {mode}\n'.format(**locals()))
|
|
f.write('build $builddir/{mode}/dist/tar/{scylla_product}-{arch}-package.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz\n'.format(**locals()))
|
|
f.write(' mode = {mode}\n'.format(**locals()))
|
|
|
|
f.write(f'build $builddir/dist/{mode}/redhat: rpmbuild $builddir/{mode}/dist/tar/{scylla_product}-unstripped-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
|
|
f.write(f' mode = {mode}\n')
|
|
f.write(f'build $builddir/dist/{mode}/debian: debbuild $builddir/{mode}/dist/tar/{scylla_product}-unstripped-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
|
|
f.write(f' mode = {mode}\n')
|
|
f.write(f'build dist-server-{mode}: phony $builddir/dist/{mode}/redhat $builddir/dist/{mode}/debian\n')
|
|
f.write(f'build dist-server-debuginfo-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-debuginfo-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
|
|
f.write(f'build dist-jmx-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz dist-jmx-rpm dist-jmx-deb\n')
|
|
f.write(f'build dist-tools-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz dist-tools-rpm dist-tools-deb\n')
|
|
f.write(f'build dist-cqlsh-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz dist-cqlsh-rpm dist-cqlsh-deb\n')
|
|
f.write(f'build dist-python3-{mode}: phony dist-python3-tar dist-python3-rpm dist-python3-deb\n')
|
|
f.write(f'build dist-unified-{mode}: phony $builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
|
|
f.write(f'build $builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz: unified $builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz $builddir/{mode}/dist/tar/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz $builddir/{mode}/dist/tar/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz $builddir/{mode}/dist/tar/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz $builddir/{mode}/dist/tar/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz | always\n')
|
|
f.write(f' mode = {mode}\n')
|
|
f.write(f'build $builddir/{mode}/dist/tar/{scylla_product}-unified-package-{scylla_version}-{scylla_release}.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
|
|
f.write(f'build $builddir/{mode}/dist/tar/{scylla_product}-unified-{arch}-package-{scylla_version}-{scylla_release}.tar.gz: copy $builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz\n')
|
|
|
|
checkheaders_mode = 'dev' if 'dev' in modes else modes.keys()[0]
|
|
f.write('build checkheaders: phony || {}\n'.format(' '.join(['$builddir/{}/{}.o'.format(checkheaders_mode, hh) for hh in headers])))
|
|
|
|
f.write(
|
|
'build build: phony {}\n'.format(' '.join([f'{mode}-build' for mode in default_modes]))
|
|
)
|
|
f.write(
|
|
'build test: phony {}\n'.format(' '.join(['{mode}-test'.format(mode=mode) for mode in default_modes]))
|
|
)
|
|
f.write(
|
|
'build check: phony {}\n'.format(' '.join(['{mode}-check'.format(mode=mode) for mode in default_modes]))
|
|
)
|
|
f.write(
|
|
'build wasm: phony {}\n'.format(' '.join([f'$builddir/{binary}' for binary in sorted(wasms)]))
|
|
)
|
|
|
|
f.write(textwrap.dedent(f'''\
|
|
build dist-unified-tar: phony {' '.join([f'$builddir/{mode}/dist/tar/{scylla_product}-unified-{scylla_version}-{scylla_release}.{arch}.tar.gz' for mode in default_modes])}
|
|
build dist-unified: phony dist-unified-tar
|
|
|
|
build dist-server-deb: phony {' '.join(['$builddir/dist/{mode}/debian'.format(mode=mode) for mode in build_modes])}
|
|
build dist-server-rpm: phony {' '.join(['$builddir/dist/{mode}/redhat'.format(mode=mode) for mode in build_modes])}
|
|
build dist-server-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-{scylla_version}-{scylla_release}.{arch}.tar.gz'.format(mode=mode, scylla_product=scylla_product, arch=arch, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
|
|
build dist-server-debuginfo: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-debuginfo-{scylla_version}-{scylla_release}.{arch}.tar.gz'.format(mode=mode, scylla_product=scylla_product, arch=arch, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
|
|
build dist-server: phony dist-server-tar dist-server-debuginfo dist-server-rpm dist-server-deb
|
|
|
|
rule build-submodule-reloc
|
|
command = cd $reloc_dir && ./reloc/build_reloc.sh --version $$(<../../build/SCYLLA-PRODUCT-FILE)-$$(sed 's/-/~/' <../../build/SCYLLA-VERSION-FILE)-$$(<../../build/SCYLLA-RELEASE-FILE) --nodeps $args
|
|
rule build-submodule-rpm
|
|
command = cd $dir && ./reloc/build_rpm.sh --reloc-pkg $artifact
|
|
rule build-submodule-deb
|
|
command = cd $dir && ./reloc/build_deb.sh --reloc-pkg $artifact
|
|
|
|
build tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz: build-submodule-reloc | build/SCYLLA-PRODUCT-FILE build/SCYLLA-VERSION-FILE build/SCYLLA-RELEASE-FILE
|
|
reloc_dir = tools/jmx
|
|
build dist-jmx-rpm: build-submodule-rpm tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
dir = tools/jmx
|
|
artifact = $builddir/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build dist-jmx-deb: build-submodule-deb tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
dir = tools/jmx
|
|
artifact = $builddir/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build dist-jmx-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz'.format(mode=mode, scylla_product=scylla_product, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
|
|
build dist-jmx: phony dist-jmx-tar dist-jmx-rpm dist-jmx-deb
|
|
|
|
build tools/java/build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz: build-submodule-reloc | build/SCYLLA-PRODUCT-FILE build/SCYLLA-VERSION-FILE build/SCYLLA-RELEASE-FILE
|
|
reloc_dir = tools/java
|
|
build dist-tools-rpm: build-submodule-rpm tools/java/build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
dir = tools/java
|
|
artifact = $builddir/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build dist-tools-deb: build-submodule-deb tools/java/build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
dir = tools/java
|
|
artifact = $builddir/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build dist-tools-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz'.format(mode=mode, scylla_product=scylla_product, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
|
|
build dist-tools: phony dist-tools-tar dist-tools-rpm dist-tools-deb
|
|
|
|
build tools/cqlsh/build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz: build-submodule-reloc | build/SCYLLA-PRODUCT-FILE build/SCYLLA-VERSION-FILE build/SCYLLA-RELEASE-FILE
|
|
reloc_dir = tools/cqlsh
|
|
build dist-cqlsh-rpm: build-submodule-rpm tools/cqlsh/build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
dir = tools/cqlsh
|
|
artifact = $builddir/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build dist-cqlsh-deb: build-submodule-deb tools/cqlsh/build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
dir = tools/cqlsh
|
|
artifact = $builddir/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build dist-cqlsh-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz'.format(mode=mode, scylla_product=scylla_product, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
|
|
build dist-cqlsh: phony dist-cqlsh-tar dist-cqlsh-rpm dist-cqlsh-deb
|
|
|
|
build tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz: build-submodule-reloc | build/SCYLLA-PRODUCT-FILE build/SCYLLA-VERSION-FILE build/SCYLLA-RELEASE-FILE
|
|
reloc_dir = tools/python3
|
|
args = --packages "{python3_dependencies}" --pip-packages "{pip_dependencies}" --pip-symlinks "{pip_symlinks}"
|
|
build dist-python3-rpm: build-submodule-rpm tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
|
|
dir = tools/python3
|
|
artifact = $builddir/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
|
|
build dist-python3-deb: build-submodule-deb tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
|
|
dir = tools/python3
|
|
artifact = $builddir/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
|
|
build dist-python3-tar: phony {' '.join(['$builddir/{mode}/dist/tar/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz'.format(mode=mode, scylla_product=scylla_product, arch=arch, scylla_version=scylla_version, scylla_release=scylla_release) for mode in default_modes])}
|
|
build dist-python3: phony dist-python3-tar dist-python3-rpm dist-python3-deb
|
|
build dist-deb: phony dist-server-deb dist-python3-deb dist-jmx-deb dist-tools-deb dist-cqlsh-deb
|
|
build dist-rpm: phony dist-server-rpm dist-python3-rpm dist-jmx-rpm dist-tools-rpm dist-cqlsh-rpm
|
|
build dist-tar: phony dist-unified-tar dist-server-tar dist-python3-tar dist-jmx-tar dist-tools-tar dist-cqlsh-tar
|
|
|
|
build dist: phony dist-unified dist-server dist-python3 dist-jmx dist-tools dist-cqlsh
|
|
'''))
|
|
|
|
f.write(textwrap.dedent(f'''\
|
|
build dist-check: phony {' '.join(['dist-check-{mode}'.format(mode=mode) for mode in default_modes])}
|
|
rule dist-check
|
|
command = ./tools/testing/dist-check/dist-check.sh --mode $mode
|
|
'''))
|
|
for mode in build_modes:
|
|
f.write(textwrap.dedent(f'''\
|
|
build $builddir/{mode}/dist/tar/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz: copy tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
|
|
build $builddir/{mode}/dist/tar/{scylla_product}-python3-package.tar.gz: copy tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
|
|
build $builddir/{mode}/dist/tar/{scylla_product}-python3-{arch}-package.tar.gz: copy tools/python3/build/{scylla_product}-python3-{scylla_version}-{scylla_release}.{arch}.tar.gz
|
|
build $builddir/{mode}/dist/tar/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz: copy tools/java/build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build $builddir/{mode}/dist/tar/{scylla_product}-tools-package.tar.gz: copy tools/java/build/{scylla_product}-tools-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build $builddir/{mode}/dist/tar/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz: copy tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build $builddir/{mode}/dist/tar/{scylla_product}-jmx-package.tar.gz: copy tools/jmx/build/{scylla_product}-jmx-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build $builddir/{mode}/dist/tar/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz: copy tools/cqlsh/build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
build $builddir/{mode}/dist/tar/{scylla_product}-cqlsh-package.tar.gz: copy tools/cqlsh/build/{scylla_product}-cqlsh-{scylla_version}-{scylla_release}.noarch.tar.gz
|
|
|
|
build {mode}-dist: phony dist-server-{mode} dist-server-debuginfo-{mode} dist-python3-{mode} dist-tools-{mode} dist-jmx-{mode} dist-unified-{mode} dist-cqlsh-{mode}
|
|
build dist-{mode}: phony {mode}-dist
|
|
build dist-check-{mode}: dist-check
|
|
mode = {mode}
|
|
'''))
|
|
|
|
f.write(textwrap.dedent('''\
|
|
rule configure
|
|
command = {python} configure.py --out=build.ninja.new $configure_args && mv build.ninja.new build.ninja
|
|
generator = 1
|
|
description = CONFIGURE $configure_args
|
|
build build.ninja {build_ninja_list}: configure | configure.py SCYLLA-VERSION-GEN {args.seastar_path}/CMakeLists.txt
|
|
rule cscope
|
|
command = find -name '*.[chS]' -o -name "*.cc" -o -name "*.hh" | cscope -bq -i-
|
|
description = CSCOPE
|
|
build cscope: cscope
|
|
rule clean
|
|
command = rm -rf build
|
|
description = CLEAN
|
|
build clean: clean
|
|
rule mode_list
|
|
command = echo {modes_list}
|
|
description = List configured modes
|
|
build mode_list: mode_list
|
|
default {modes_list}
|
|
''').format(modes_list=' '.join(default_modes), build_ninja_list=' '.join([f'build/{mode}/{dir}/build.ninja' for mode in build_modes for dir in ['seastar']]), **globals()))
|
|
unit_test_list = set(test for test in build_artifacts if test in set(tests))
|
|
f.write(textwrap.dedent('''\
|
|
rule unit_test_list
|
|
command = /usr/bin/env echo -e '{unit_test_list}'
|
|
description = List configured unit tests
|
|
build unit_test_list: unit_test_list
|
|
''').format(unit_test_list="\\n".join(sorted(unit_test_list))))
|
|
f.write(textwrap.dedent('''\
|
|
build always: phony
|
|
rule scylla_version_gen
|
|
command = ./SCYLLA-VERSION-GEN
|
|
build $builddir/SCYLLA-RELEASE-FILE $builddir/SCYLLA-VERSION-FILE: scylla_version_gen
|
|
rule debian_files_gen
|
|
command = ./dist/debian/debian_files_gen.py
|
|
build $builddir/debian/debian: debian_files_gen | always
|
|
rule extract_node_exporter
|
|
command = tar -C build -xvpf {node_exporter_filename} --no-same-owner && rm -rfv build/node_exporter && mv -v build/{node_exporter_dirname} build/node_exporter
|
|
build $builddir/node_exporter/node_exporter: extract_node_exporter | always
|
|
build $builddir/node_exporter/node_exporter.stripped: strip $builddir/node_exporter/node_exporter
|
|
build $builddir/node_exporter/node_exporter.debug: phony $builddir/node_exporter/node_exporter.stripped
|
|
rule print_help
|
|
command = ./scripts/build-help.sh
|
|
build help: print_help | always
|
|
''').format(**globals()))
|
|
|
|
|
|
def create_build_system(args):
|
|
check_for_minimal_compiler_version(args.cxx)
|
|
check_for_boost(args.cxx)
|
|
check_for_lz4(args.cxx, args.user_cflags)
|
|
|
|
os.makedirs(outdir, exist_ok=True)
|
|
|
|
if not args.dist_only:
|
|
# args.buildfile builds seastar with the rules of
|
|
# {outdir}/{mode}/seastar/build.ninja, and
|
|
# {outdir}/{mode}/seastar/seastar.pc is queried for building flags
|
|
for mode, mode_config in build_modes.items():
|
|
configure_seastar(outdir, mode, mode_config)
|
|
|
|
ninja = find_ninja()
|
|
with open(args.buildfile, 'w') as f:
|
|
scylla_product, scylla_version, scylla_release = generate_version(args.date_stamp)
|
|
arch = platform.machine()
|
|
write_build_file(f,
|
|
arch,
|
|
ninja,
|
|
scylla_product,
|
|
scylla_version,
|
|
scylla_release,
|
|
args)
|
|
generate_compdb('compile_commands.json', ninja, args.buildfile, selected_modes)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
create_build_system(args)
|