1005 lines
41 KiB
Python
Executable File
1005 lines
41 KiB
Python
Executable File
#!/usr/bin/python3
|
|
#
|
|
# Copyright (C) 2015 ScyllaDB
|
|
#
|
|
|
|
#
|
|
# This file is part of Scylla.
|
|
#
|
|
# Scylla is free software: you can redistribute it and/or modify
|
|
# it under the terms of the GNU Affero General Public License as published by
|
|
# the Free Software Foundation, either version 3 of the License, or
|
|
# (at your option) any later version.
|
|
#
|
|
# Scylla is distributed in the hope that it will be useful,
|
|
# but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
# GNU General Public License for more details.
|
|
#
|
|
# You should have received a copy of the GNU General Public License
|
|
# along with Scylla. If not, see <http://www.gnu.org/licenses/>.
|
|
#
|
|
|
|
import os, os.path, textwrap, argparse, sys, shlex, subprocess, tempfile, re
|
|
from distutils.spawn import find_executable
|
|
|
|
configure_args = str.join(' ', [shlex.quote(x) for x in sys.argv[1:]])
|
|
|
|
for line in open('/etc/os-release'):
|
|
key, _, value = line.partition('=')
|
|
value = value.strip().strip('"')
|
|
if key == 'ID':
|
|
os_ids = [value]
|
|
if key == 'ID_LIKE':
|
|
os_ids += value.split(' ')
|
|
|
|
# distribution "internationalization", converting package names.
|
|
# Fedora name is key, values is distro -> package name dict.
|
|
i18n_xlat = {
|
|
'boost-devel': {
|
|
'debian': 'libboost-dev',
|
|
'ubuntu': 'libboost-dev (libboost1.55-dev on 14.04)',
|
|
},
|
|
}
|
|
|
|
def pkgname(name):
|
|
if name in i18n_xlat:
|
|
dict = i18n_xlat[name]
|
|
for id in os_ids:
|
|
if id in dict:
|
|
return dict[id]
|
|
return name
|
|
|
|
def get_flags():
|
|
with open('/proc/cpuinfo') as f:
|
|
for line in f:
|
|
if line.strip():
|
|
if line.rstrip('\n').startswith('flags'):
|
|
return re.sub(r'^flags\s+: ', '', line).split()
|
|
|
|
def add_tristate(arg_parser, name, dest, help):
|
|
arg_parser.add_argument('--enable-' + name, dest = dest, action = 'store_true', default = None,
|
|
help = 'Enable ' + help)
|
|
arg_parser.add_argument('--disable-' + name, dest = dest, action = 'store_false', default = None,
|
|
help = 'Disable ' + help)
|
|
|
|
def apply_tristate(var, test, note, missing):
|
|
if (var is None) or var:
|
|
if test():
|
|
return True
|
|
elif var == True:
|
|
print(missing)
|
|
sys.exit(1)
|
|
else:
|
|
print(note)
|
|
return False
|
|
return False
|
|
|
|
def have_pkg(package):
|
|
return subprocess.call(['pkg-config', package]) == 0
|
|
|
|
def pkg_config(option, package):
|
|
output = subprocess.check_output(['pkg-config', option, package])
|
|
return output.decode('utf-8').strip()
|
|
|
|
def try_compile(compiler, source = '', flags = []):
|
|
with tempfile.NamedTemporaryFile() as sfile:
|
|
sfile.file.write(bytes(source, 'utf-8'))
|
|
sfile.file.flush()
|
|
return subprocess.call([compiler, '-x', 'c++', '-o', '/dev/null', '-c', sfile.name] + flags,
|
|
stdout = subprocess.DEVNULL,
|
|
stderr = subprocess.DEVNULL) == 0
|
|
|
|
def warning_supported(warning, compiler):
|
|
# gcc ignores -Wno-x even if it is not supported
|
|
adjusted = re.sub('^-Wno-', '-W', warning)
|
|
return try_compile(flags = ['-Werror', adjusted], compiler = compiler)
|
|
|
|
def debug_flag(compiler):
|
|
src_with_auto = textwrap.dedent('''\
|
|
template <typename T>
|
|
struct x { auto f() {} };
|
|
|
|
x<int> a;
|
|
''')
|
|
if try_compile(source = src_with_auto, flags = ['-g', '-std=gnu++1y'], compiler = compiler):
|
|
return '-g'
|
|
else:
|
|
print('Note: debug information disabled; upgrade your compiler')
|
|
return ''
|
|
|
|
def maybe_static(flag, libs):
|
|
if flag and not args.static:
|
|
libs = '-Wl,-Bstatic {} -Wl,-Bdynamic'.format(libs)
|
|
return libs
|
|
|
|
class Thrift(object):
|
|
def __init__(self, source, service):
|
|
self.source = source
|
|
self.service = service
|
|
def generated(self, gen_dir):
|
|
basename = os.path.splitext(os.path.basename(self.source))[0]
|
|
files = [basename + '_' + ext
|
|
for ext in ['types.cpp', 'types.h', 'constants.cpp', 'constants.h']]
|
|
files += [self.service + ext
|
|
for ext in ['.cpp', '.h']]
|
|
return [os.path.join(gen_dir, file) for file in files]
|
|
def headers(self, gen_dir):
|
|
return [x for x in self.generated(gen_dir) if x.endswith('.h')]
|
|
def sources(self, gen_dir):
|
|
return [x for x in self.generated(gen_dir) if x.endswith('.cpp')]
|
|
def objects(self, gen_dir):
|
|
return [x.replace('.cpp', '.o') for x in self.sources(gen_dir)]
|
|
def endswith(self, end):
|
|
return self.source.endswith(end)
|
|
|
|
class Antlr3Grammar(object):
|
|
def __init__(self, source):
|
|
self.source = source
|
|
def generated(self, gen_dir):
|
|
basename = os.path.splitext(self.source)[0]
|
|
files = [basename + ext
|
|
for ext in ['Lexer.cpp', 'Lexer.hpp', 'Parser.cpp', 'Parser.hpp']]
|
|
return [os.path.join(gen_dir, file) for file in files]
|
|
def headers(self, gen_dir):
|
|
return [x for x in self.generated(gen_dir) if x.endswith('.hpp')]
|
|
def sources(self, gen_dir):
|
|
return [x for x in self.generated(gen_dir) if x.endswith('.cpp')]
|
|
def objects(self, gen_dir):
|
|
return [x.replace('.cpp', '.o') for x in self.sources(gen_dir)]
|
|
def endswith(self, end):
|
|
return self.source.endswith(end)
|
|
|
|
modes = {
|
|
'debug': {
|
|
'sanitize': '-fsanitize=address -fsanitize=leak -fsanitize=undefined',
|
|
'sanitize_libs': '-lasan -lubsan',
|
|
'opt': '-O0 -DDEBUG -DDEBUG_SHARED_PTR -DDEFAULT_ALLOCATOR',
|
|
'libs': '',
|
|
},
|
|
'release': {
|
|
'sanitize': '',
|
|
'sanitize_libs': '',
|
|
'opt': '-O2',
|
|
'libs': '',
|
|
},
|
|
}
|
|
|
|
scylla_tests = [
|
|
'tests/mutation_test',
|
|
'tests/streamed_mutation_test',
|
|
'tests/schema_registry_test',
|
|
'tests/canonical_mutation_test',
|
|
'tests/range_test',
|
|
'tests/types_test',
|
|
'tests/keys_test',
|
|
'tests/partitioner_test',
|
|
'tests/frozen_mutation_test',
|
|
'tests/perf/perf_mutation',
|
|
'tests/lsa_async_eviction_test',
|
|
'tests/lsa_sync_eviction_test',
|
|
'tests/row_cache_alloc_stress',
|
|
'tests/perf_row_cache_update',
|
|
'tests/perf/perf_hash',
|
|
'tests/perf/perf_cql_parser',
|
|
'tests/perf/perf_simple_query',
|
|
'tests/perf/perf_fast_forward',
|
|
'tests/memory_footprint',
|
|
'tests/perf/perf_sstable',
|
|
'tests/cql_query_test',
|
|
'tests/storage_proxy_test',
|
|
'tests/schema_change_test',
|
|
'tests/mutation_reader_test',
|
|
'tests/mutation_query_test',
|
|
'tests/row_cache_test',
|
|
'tests/test-serialization',
|
|
'tests/sstable_test',
|
|
'tests/sstable_mutation_test',
|
|
'tests/memtable_test',
|
|
'tests/commitlog_test',
|
|
'tests/cartesian_product_test',
|
|
'tests/hash_test',
|
|
'tests/map_difference_test',
|
|
'tests/message',
|
|
'tests/gossip',
|
|
'tests/gossip_test',
|
|
'tests/compound_test',
|
|
'tests/config_test',
|
|
'tests/gossiping_property_file_snitch_test',
|
|
'tests/ec2_snitch_test',
|
|
'tests/snitch_reset_test',
|
|
'tests/network_topology_strategy_test',
|
|
'tests/query_processor_test',
|
|
'tests/batchlog_manager_test',
|
|
'tests/bytes_ostream_test',
|
|
'tests/UUID_test',
|
|
'tests/murmur_hash_test',
|
|
'tests/allocation_strategy_test',
|
|
'tests/logalloc_test',
|
|
'tests/managed_vector_test',
|
|
'tests/crc_test',
|
|
'tests/flush_queue_test',
|
|
'tests/dynamic_bitset_test',
|
|
'tests/auth_test',
|
|
'tests/idl_test',
|
|
'tests/range_tombstone_list_test',
|
|
'tests/anchorless_list_test',
|
|
'tests/database_test',
|
|
'tests/nonwrapping_range_test',
|
|
'tests/input_stream_test',
|
|
'tests/sstable_atomic_deletion_test',
|
|
'tests/virtual_reader_test',
|
|
'tests/view_schema_test',
|
|
'tests/counter_test',
|
|
'tests/cell_locker_test',
|
|
]
|
|
|
|
apps = [
|
|
'scylla',
|
|
]
|
|
|
|
tests = scylla_tests
|
|
|
|
other = [
|
|
'iotune',
|
|
]
|
|
|
|
all_artifacts = apps + tests + other
|
|
|
|
arg_parser = argparse.ArgumentParser('Configure scylla')
|
|
arg_parser.add_argument('--static', dest = 'static', action = 'store_const', default = '',
|
|
const = '-static',
|
|
help = 'Static link (useful for running on hosts outside the build environment')
|
|
arg_parser.add_argument('--pie', dest = 'pie', action = 'store_true',
|
|
help = 'Build position-independent executable (PIE)')
|
|
arg_parser.add_argument('--so', dest = 'so', action = 'store_true',
|
|
help = 'Build shared object (SO) instead of executable')
|
|
arg_parser.add_argument('--mode', action='store', choices=list(modes.keys()) + ['all'], default='all')
|
|
arg_parser.add_argument('--with', dest='artifacts', action='append', choices=all_artifacts, default=[])
|
|
arg_parser.add_argument('--cflags', action = 'store', dest = 'user_cflags', default = '',
|
|
help = 'Extra flags for the C++ compiler')
|
|
arg_parser.add_argument('--ldflags', action = 'store', dest = 'user_ldflags', default = '',
|
|
help = 'Extra flags for the linker')
|
|
arg_parser.add_argument('--compiler', action = 'store', dest = 'cxx', default = 'g++',
|
|
help = 'C++ compiler path')
|
|
arg_parser.add_argument('--c-compiler', action='store', dest='cc', default='gcc',
|
|
help='C compiler path')
|
|
arg_parser.add_argument('--with-osv', action = 'store', dest = 'with_osv', default = '',
|
|
help = 'Shortcut for compile for OSv')
|
|
arg_parser.add_argument('--enable-dpdk', action = 'store_true', dest = 'dpdk', default = False,
|
|
help = 'Enable dpdk (from seastar dpdk sources)')
|
|
arg_parser.add_argument('--dpdk-target', action = 'store', dest = 'dpdk_target', default = '',
|
|
help = 'Path to DPDK SDK target location (e.g. <DPDK SDK dir>/x86_64-native-linuxapp-gcc)')
|
|
arg_parser.add_argument('--debuginfo', action = 'store', dest = 'debuginfo', type = int, default = 1,
|
|
help = 'Enable(1)/disable(0)compiler debug information generation')
|
|
arg_parser.add_argument('--static-stdc++', dest = 'staticcxx', action = 'store_true',
|
|
help = 'Link libgcc and libstdc++ statically')
|
|
arg_parser.add_argument('--static-thrift', dest = 'staticthrift', action = 'store_true',
|
|
help = 'Link libthrift statically')
|
|
arg_parser.add_argument('--static-boost', dest = 'staticboost', action = 'store_true',
|
|
help = 'Link boost statically')
|
|
arg_parser.add_argument('--tests-debuginfo', action = 'store', dest = 'tests_debuginfo', type = int, default = 0,
|
|
help = 'Enable(1)/disable(0)compiler debug information generation for tests')
|
|
arg_parser.add_argument('--python', action = 'store', dest = 'python', default = 'python3',
|
|
help = 'Python3 path')
|
|
add_tristate(arg_parser, name = 'hwloc', dest = 'hwloc', help = 'hwloc support')
|
|
add_tristate(arg_parser, name = 'xen', dest = 'xen', help = 'Xen support')
|
|
arg_parser.add_argument('--enable-gcc6-concepts', dest='gcc6_concepts', action='store_true', default=False,
|
|
help='enable experimental support for C++ Concepts as implemented in GCC 6')
|
|
args = arg_parser.parse_args()
|
|
|
|
defines = []
|
|
|
|
extra_cxxflags = {}
|
|
|
|
cassandra_interface = Thrift(source = 'interface/cassandra.thrift', service = 'Cassandra')
|
|
|
|
scylla_core = (['database.cc',
|
|
'schema.cc',
|
|
'frozen_schema.cc',
|
|
'schema_registry.cc',
|
|
'bytes.cc',
|
|
'mutation.cc',
|
|
'streamed_mutation.cc',
|
|
'partition_version.cc',
|
|
'row_cache.cc',
|
|
'canonical_mutation.cc',
|
|
'frozen_mutation.cc',
|
|
'memtable.cc',
|
|
'schema_mutations.cc',
|
|
'release.cc',
|
|
'supervisor.cc',
|
|
'utils/logalloc.cc',
|
|
'utils/large_bitset.cc',
|
|
'mutation_partition.cc',
|
|
'mutation_partition_view.cc',
|
|
'mutation_partition_serializer.cc',
|
|
'mutation_reader.cc',
|
|
'mutation_query.cc',
|
|
'keys.cc',
|
|
'counters.cc',
|
|
'sstables/sstables.cc',
|
|
'sstables/compress.cc',
|
|
'sstables/row.cc',
|
|
'sstables/partition.cc',
|
|
'sstables/filter.cc',
|
|
'sstables/compaction.cc',
|
|
'sstables/compaction_strategy.cc',
|
|
'sstables/compaction_manager.cc',
|
|
'sstables/atomic_deletion.cc',
|
|
'transport/event.cc',
|
|
'transport/event_notifier.cc',
|
|
'transport/server.cc',
|
|
'cql3/abstract_marker.cc',
|
|
'cql3/attributes.cc',
|
|
'cql3/cf_name.cc',
|
|
'cql3/cql3_type.cc',
|
|
'cql3/operation.cc',
|
|
'cql3/index_name.cc',
|
|
'cql3/keyspace_element_name.cc',
|
|
'cql3/lists.cc',
|
|
'cql3/sets.cc',
|
|
'cql3/maps.cc',
|
|
'cql3/functions/functions.cc',
|
|
'cql3/statements/cf_prop_defs.cc',
|
|
'cql3/statements/cf_statement.cc',
|
|
'cql3/statements/authentication_statement.cc',
|
|
'cql3/statements/create_keyspace_statement.cc',
|
|
'cql3/statements/create_table_statement.cc',
|
|
'cql3/statements/create_view_statement.cc',
|
|
'cql3/statements/create_type_statement.cc',
|
|
'cql3/statements/create_user_statement.cc',
|
|
'cql3/statements/drop_keyspace_statement.cc',
|
|
'cql3/statements/drop_table_statement.cc',
|
|
'cql3/statements/drop_view_statement.cc',
|
|
'cql3/statements/drop_type_statement.cc',
|
|
'cql3/statements/schema_altering_statement.cc',
|
|
'cql3/statements/ks_prop_defs.cc',
|
|
'cql3/statements/modification_statement.cc',
|
|
'cql3/statements/parsed_statement.cc',
|
|
'cql3/statements/property_definitions.cc',
|
|
'cql3/statements/update_statement.cc',
|
|
'cql3/statements/delete_statement.cc',
|
|
'cql3/statements/batch_statement.cc',
|
|
'cql3/statements/select_statement.cc',
|
|
'cql3/statements/use_statement.cc',
|
|
'cql3/statements/index_prop_defs.cc',
|
|
'cql3/statements/index_target.cc',
|
|
'cql3/statements/create_index_statement.cc',
|
|
'cql3/statements/truncate_statement.cc',
|
|
'cql3/statements/alter_table_statement.cc',
|
|
'cql3/statements/alter_view_statement.cc',
|
|
'cql3/statements/alter_user_statement.cc',
|
|
'cql3/statements/drop_user_statement.cc',
|
|
'cql3/statements/list_users_statement.cc',
|
|
'cql3/statements/authorization_statement.cc',
|
|
'cql3/statements/permission_altering_statement.cc',
|
|
'cql3/statements/list_permissions_statement.cc',
|
|
'cql3/statements/grant_statement.cc',
|
|
'cql3/statements/revoke_statement.cc',
|
|
'cql3/statements/alter_type_statement.cc',
|
|
'cql3/statements/alter_keyspace_statement.cc',
|
|
'cql3/update_parameters.cc',
|
|
'cql3/ut_name.cc',
|
|
'cql3/user_options.cc',
|
|
'thrift/handler.cc',
|
|
'thrift/server.cc',
|
|
'thrift/thrift_validation.cc',
|
|
'utils/runtime.cc',
|
|
'utils/murmur_hash.cc',
|
|
'utils/uuid.cc',
|
|
'utils/big_decimal.cc',
|
|
'types.cc',
|
|
'validation.cc',
|
|
'service/priority_manager.cc',
|
|
'service/migration_manager.cc',
|
|
'service/storage_proxy.cc',
|
|
'cql3/operator.cc',
|
|
'cql3/relation.cc',
|
|
'cql3/column_identifier.cc',
|
|
'cql3/column_specification.cc',
|
|
'cql3/constants.cc',
|
|
'cql3/query_processor.cc',
|
|
'cql3/query_options.cc',
|
|
'cql3/single_column_relation.cc',
|
|
'cql3/token_relation.cc',
|
|
'cql3/column_condition.cc',
|
|
'cql3/user_types.cc',
|
|
'cql3/untyped_result_set.cc',
|
|
'cql3/selection/abstract_function_selector.cc',
|
|
'cql3/selection/simple_selector.cc',
|
|
'cql3/selection/selectable.cc',
|
|
'cql3/selection/selector_factories.cc',
|
|
'cql3/selection/selection.cc',
|
|
'cql3/selection/selector.cc',
|
|
'cql3/restrictions/statement_restrictions.cc',
|
|
'cql3/result_set.cc',
|
|
'db/consistency_level.cc',
|
|
'db/system_keyspace.cc',
|
|
'db/schema_tables.cc',
|
|
'db/commitlog/commitlog.cc',
|
|
'db/commitlog/commitlog_replayer.cc',
|
|
'db/commitlog/commitlog_entry.cc',
|
|
'db/config.cc',
|
|
'db/index/secondary_index.cc',
|
|
'db/marshal/type_parser.cc',
|
|
'db/batchlog_manager.cc',
|
|
'db/view/view.cc',
|
|
'io/io.cc',
|
|
'utils/utils.cc',
|
|
'utils/UUID_gen.cc',
|
|
'utils/i_filter.cc',
|
|
'utils/bloom_filter.cc',
|
|
'utils/bloom_calculations.cc',
|
|
'utils/rate_limiter.cc',
|
|
'utils/file_lock.cc',
|
|
'utils/dynamic_bitset.cc',
|
|
'utils/managed_bytes.cc',
|
|
'utils/exceptions.cc',
|
|
'gms/version_generator.cc',
|
|
'gms/versioned_value.cc',
|
|
'gms/gossiper.cc',
|
|
'gms/failure_detector.cc',
|
|
'gms/gossip_digest_syn.cc',
|
|
'gms/gossip_digest_ack.cc',
|
|
'gms/gossip_digest_ack2.cc',
|
|
'gms/endpoint_state.cc',
|
|
'gms/application_state.cc',
|
|
'gms/inet_address.cc',
|
|
'dht/i_partitioner.cc',
|
|
'dht/murmur3_partitioner.cc',
|
|
'dht/byte_ordered_partitioner.cc',
|
|
'dht/random_partitioner.cc',
|
|
'dht/boot_strapper.cc',
|
|
'dht/range_streamer.cc',
|
|
'unimplemented.cc',
|
|
'query.cc',
|
|
'query-result-set.cc',
|
|
'locator/abstract_replication_strategy.cc',
|
|
'locator/simple_strategy.cc',
|
|
'locator/local_strategy.cc',
|
|
'locator/network_topology_strategy.cc',
|
|
'locator/everywhere_replication_strategy.cc',
|
|
'locator/token_metadata.cc',
|
|
'locator/locator.cc',
|
|
'locator/snitch_base.cc',
|
|
'locator/simple_snitch.cc',
|
|
'locator/rack_inferring_snitch.cc',
|
|
'locator/gossiping_property_file_snitch.cc',
|
|
'locator/production_snitch_base.cc',
|
|
'locator/ec2_snitch.cc',
|
|
'locator/ec2_multi_region_snitch.cc',
|
|
'message/messaging_service.cc',
|
|
'service/client_state.cc',
|
|
'service/migration_task.cc',
|
|
'service/storage_service.cc',
|
|
'service/load_broadcaster.cc',
|
|
'service/pager/paging_state.cc',
|
|
'service/pager/query_pagers.cc',
|
|
'streaming/stream_task.cc',
|
|
'streaming/stream_session.cc',
|
|
'streaming/stream_request.cc',
|
|
'streaming/stream_summary.cc',
|
|
'streaming/stream_transfer_task.cc',
|
|
'streaming/stream_receive_task.cc',
|
|
'streaming/stream_plan.cc',
|
|
'streaming/progress_info.cc',
|
|
'streaming/session_info.cc',
|
|
'streaming/stream_coordinator.cc',
|
|
'streaming/stream_manager.cc',
|
|
'streaming/stream_result_future.cc',
|
|
'streaming/stream_session_state.cc',
|
|
'gc_clock.cc',
|
|
'partition_slice_builder.cc',
|
|
'init.cc',
|
|
'lister.cc',
|
|
'repair/repair.cc',
|
|
'exceptions/exceptions.cc',
|
|
'auth/auth.cc',
|
|
'auth/authenticated_user.cc',
|
|
'auth/authenticator.cc',
|
|
'auth/authorizer.cc',
|
|
'auth/default_authorizer.cc',
|
|
'auth/data_resource.cc',
|
|
'auth/password_authenticator.cc',
|
|
'auth/permission.cc',
|
|
'tracing/tracing.cc',
|
|
'tracing/trace_keyspace_helper.cc',
|
|
'tracing/trace_state.cc',
|
|
'range_tombstone.cc',
|
|
'range_tombstone_list.cc',
|
|
'disk-error-handler.cc'
|
|
]
|
|
+ [Antlr3Grammar('cql3/Cql.g')]
|
|
+ [Thrift('interface/cassandra.thrift', 'Cassandra')]
|
|
)
|
|
|
|
api = ['api/api.cc',
|
|
'api/api-doc/storage_service.json',
|
|
'api/api-doc/lsa.json',
|
|
'api/storage_service.cc',
|
|
'api/api-doc/commitlog.json',
|
|
'api/commitlog.cc',
|
|
'api/api-doc/gossiper.json',
|
|
'api/gossiper.cc',
|
|
'api/api-doc/failure_detector.json',
|
|
'api/failure_detector.cc',
|
|
'api/api-doc/column_family.json',
|
|
'api/column_family.cc',
|
|
'api/messaging_service.cc',
|
|
'api/api-doc/messaging_service.json',
|
|
'api/api-doc/storage_proxy.json',
|
|
'api/storage_proxy.cc',
|
|
'api/api-doc/cache_service.json',
|
|
'api/cache_service.cc',
|
|
'api/api-doc/collectd.json',
|
|
'api/collectd.cc',
|
|
'api/api-doc/endpoint_snitch_info.json',
|
|
'api/endpoint_snitch.cc',
|
|
'api/api-doc/compaction_manager.json',
|
|
'api/compaction_manager.cc',
|
|
'api/api-doc/hinted_handoff.json',
|
|
'api/hinted_handoff.cc',
|
|
'api/api-doc/utils.json',
|
|
'api/lsa.cc',
|
|
'api/api-doc/stream_manager.json',
|
|
'api/stream_manager.cc',
|
|
'api/api-doc/system.json',
|
|
'api/system.cc'
|
|
]
|
|
|
|
idls = ['idl/gossip_digest.idl.hh',
|
|
'idl/uuid.idl.hh',
|
|
'idl/range.idl.hh',
|
|
'idl/keys.idl.hh',
|
|
'idl/read_command.idl.hh',
|
|
'idl/token.idl.hh',
|
|
'idl/ring_position.idl.hh',
|
|
'idl/result.idl.hh',
|
|
'idl/frozen_mutation.idl.hh',
|
|
'idl/reconcilable_result.idl.hh',
|
|
'idl/streaming.idl.hh',
|
|
'idl/paging_state.idl.hh',
|
|
'idl/frozen_schema.idl.hh',
|
|
'idl/partition_checksum.idl.hh',
|
|
'idl/replay_position.idl.hh',
|
|
'idl/truncation_record.idl.hh',
|
|
'idl/mutation.idl.hh',
|
|
'idl/query.idl.hh',
|
|
'idl/idl_test.idl.hh',
|
|
'idl/commitlog.idl.hh',
|
|
'idl/tracing.idl.hh',
|
|
'idl/consistency_level.idl.hh',
|
|
]
|
|
|
|
scylla_tests_dependencies = scylla_core + api + idls + [
|
|
'tests/cql_test_env.cc',
|
|
'tests/cql_assertions.cc',
|
|
'tests/result_set_assertions.cc',
|
|
'tests/mutation_source_test.cc',
|
|
]
|
|
|
|
scylla_tests_seastar_deps = [
|
|
'seastar/tests/test-utils.cc',
|
|
'seastar/tests/test_runner.cc',
|
|
]
|
|
|
|
deps = {
|
|
'scylla': idls + ['main.cc'] + scylla_core + api,
|
|
}
|
|
|
|
pure_boost_tests = set([
|
|
'tests/partitioner_test',
|
|
'tests/map_difference_test',
|
|
'tests/keys_test',
|
|
'tests/compound_test',
|
|
'tests/range_tombstone_list_test',
|
|
'tests/anchorless_list_test',
|
|
'tests/nonwrapping_range_test',
|
|
'tests/test-serialization',
|
|
'tests/range_test',
|
|
'tests/crc_test',
|
|
'tests/managed_vector_test',
|
|
'tests/dynamic_bitset_test',
|
|
'tests/idl_test',
|
|
'tests/cartesian_product_test',
|
|
])
|
|
|
|
tests_not_using_seastar_test_framework = set([
|
|
'tests/perf/perf_mutation',
|
|
'tests/lsa_async_eviction_test',
|
|
'tests/lsa_sync_eviction_test',
|
|
'tests/row_cache_alloc_stress',
|
|
'tests/perf_row_cache_update',
|
|
'tests/perf/perf_hash',
|
|
'tests/perf/perf_cql_parser',
|
|
'tests/message',
|
|
'tests/perf/perf_simple_query',
|
|
'tests/perf/perf_fast_forward',
|
|
'tests/memory_footprint',
|
|
'tests/gossip',
|
|
'tests/perf/perf_sstable',
|
|
]) | pure_boost_tests
|
|
|
|
for t in tests_not_using_seastar_test_framework:
|
|
if not t in scylla_tests:
|
|
raise Exception("Test %s not found in scylla_tests" % (t))
|
|
|
|
for t in scylla_tests:
|
|
deps[t] = [t + '.cc']
|
|
if t not in tests_not_using_seastar_test_framework:
|
|
deps[t] += scylla_tests_dependencies
|
|
deps[t] += scylla_tests_seastar_deps
|
|
else:
|
|
deps[t] += scylla_core + api + idls + ['tests/cql_test_env.cc']
|
|
|
|
deps['tests/sstable_test'] += ['tests/sstable_datafile_test.cc']
|
|
|
|
deps['tests/bytes_ostream_test'] = ['tests/bytes_ostream_test.cc']
|
|
deps['tests/input_stream_test'] = ['tests/input_stream_test.cc']
|
|
deps['tests/UUID_test'] = ['utils/UUID_gen.cc', 'tests/UUID_test.cc', 'utils/uuid.cc']
|
|
deps['tests/murmur_hash_test'] = ['bytes.cc', 'utils/murmur_hash.cc', 'tests/murmur_hash_test.cc']
|
|
deps['tests/allocation_strategy_test'] = ['tests/allocation_strategy_test.cc', 'utils/logalloc.cc', 'utils/dynamic_bitset.cc']
|
|
deps['tests/anchorless_list_test'] = ['tests/anchorless_list_test.cc']
|
|
|
|
warnings = [
|
|
'-Wno-mismatched-tags', # clang-only
|
|
'-Wno-maybe-uninitialized', # false positives on gcc 5
|
|
'-Wno-tautological-compare',
|
|
'-Wno-parentheses-equality',
|
|
'-Wno-c++11-narrowing',
|
|
'-Wno-c++1z-extensions',
|
|
'-Wno-sometimes-uninitialized',
|
|
'-Wno-return-stack-address',
|
|
'-Wno-missing-braces',
|
|
]
|
|
|
|
warnings = [w
|
|
for w in warnings
|
|
if warning_supported(warning = w, compiler = args.cxx)]
|
|
|
|
warnings = ' '.join(warnings + ['-Wno-error=deprecated-declarations'])
|
|
|
|
dbgflag = debug_flag(args.cxx) if args.debuginfo else ''
|
|
tests_link_rule = 'link' if args.tests_debuginfo else 'link_stripped'
|
|
|
|
if args.so:
|
|
args.pie = '-shared'
|
|
args.fpie = '-fpic'
|
|
elif args.pie:
|
|
args.pie = '-pie'
|
|
args.fpie = '-fpie'
|
|
else:
|
|
args.pie = ''
|
|
args.fpie = ''
|
|
|
|
# a list element means a list of alternative packages to consider
|
|
# the first element becomes the HAVE_pkg define
|
|
# a string element is a package name with no alternatives
|
|
optional_packages = [['libsystemd', 'libsystemd-daemon']]
|
|
pkgs = []
|
|
|
|
def setup_first_pkg_of_list(pkglist):
|
|
# The HAVE_pkg symbol is taken from the first alternative
|
|
upkg = pkglist[0].upper().replace('-', '_')
|
|
for pkg in pkglist:
|
|
if have_pkg(pkg):
|
|
pkgs.append(pkg)
|
|
defines.append('HAVE_{}=1'.format(upkg))
|
|
return True
|
|
return False
|
|
|
|
for pkglist in optional_packages:
|
|
if isinstance(pkglist, str):
|
|
pkglist = [pkglist]
|
|
if not setup_first_pkg_of_list(pkglist):
|
|
if len(pkglist) == 1:
|
|
print('Missing optional package {pkglist[0]}'.format(**locals()))
|
|
else:
|
|
alternatives = ':'.join(pkglist[1:])
|
|
print('Missing optional package {pkglist[0]} (or alteratives {alternatives})'.format(**locals()))
|
|
|
|
if not try_compile(compiler=args.cxx, source='#include <boost/version.hpp>'):
|
|
print('Boost not installed. Please install {}.'.format(pkgname("boost-devel")))
|
|
sys.exit(1)
|
|
|
|
if not try_compile(compiler=args.cxx, source='''\
|
|
#include <boost/version.hpp>
|
|
#if BOOST_VERSION < 105500
|
|
#error Boost version too low
|
|
#endif
|
|
'''):
|
|
print('Installed boost version too old. Please update {}.'.format(pkgname("boost-devel")))
|
|
sys.exit(1)
|
|
|
|
defines = ' '.join(['-D' + d for d in defines])
|
|
|
|
globals().update(vars(args))
|
|
|
|
total_memory = os.sysconf('SC_PAGE_SIZE') * os.sysconf('SC_PHYS_PAGES')
|
|
link_pool_depth = max(int(total_memory / 7e9), 1)
|
|
|
|
build_modes = modes if args.mode == 'all' else [args.mode]
|
|
build_artifacts = all_artifacts if not args.artifacts else args.artifacts
|
|
|
|
status = subprocess.call("./SCYLLA-VERSION-GEN")
|
|
if status != 0:
|
|
print('Version file generation failed')
|
|
sys.exit(1)
|
|
|
|
file = open('build/SCYLLA-VERSION-FILE', 'r')
|
|
scylla_version = file.read().strip()
|
|
file = open('build/SCYLLA-RELEASE-FILE', 'r')
|
|
scylla_release = file.read().strip()
|
|
|
|
extra_cxxflags["release.cc"] = "-DSCYLLA_VERSION=\"\\\"" + scylla_version + "\\\"\" -DSCYLLA_RELEASE=\"\\\"" + scylla_release + "\\\"\""
|
|
|
|
seastar_flags = ['--disable-xen']
|
|
if args.dpdk:
|
|
# fake dependencies on dpdk, so that it is built before anything else
|
|
seastar_flags += ['--enable-dpdk']
|
|
elif args.dpdk_target:
|
|
seastar_flags += ['--dpdk-target', args.dpdk_target]
|
|
if args.staticcxx:
|
|
seastar_flags += ['--static-stdc++']
|
|
if args.staticboost:
|
|
seastar_flags += ['--static-boost']
|
|
if args.gcc6_concepts:
|
|
seastar_flags += ['--enable-gcc6-concepts']
|
|
|
|
seastar_cflags = args.user_cflags + " -march=nehalem"
|
|
seastar_flags += ['--compiler', args.cxx, '--c-compiler', args.cc, '--cflags=%s' % (seastar_cflags)]
|
|
|
|
status = subprocess.call([python, './configure.py'] + seastar_flags, cwd = 'seastar')
|
|
|
|
if status != 0:
|
|
print('Seastar configuration failed')
|
|
sys.exit(1)
|
|
|
|
|
|
pc = { mode : 'build/{}/seastar.pc'.format(mode) for mode in build_modes }
|
|
ninja = find_executable('ninja') or find_executable('ninja-build')
|
|
if not ninja:
|
|
print('Ninja executable (ninja or ninja-build) not found on PATH\n')
|
|
sys.exit(1)
|
|
status = subprocess.call([ninja] + list(pc.values()), cwd = 'seastar')
|
|
if status:
|
|
print('Failed to generate {}\n'.format(pc))
|
|
sys.exit(1)
|
|
|
|
for mode in build_modes:
|
|
cfg = dict([line.strip().split(': ', 1)
|
|
for line in open('seastar/' + pc[mode])
|
|
if ': ' in line])
|
|
if args.staticcxx:
|
|
cfg['Libs'] = cfg['Libs'].replace('-lstdc++ ', '')
|
|
modes[mode]['seastar_cflags'] = cfg['Cflags']
|
|
modes[mode]['seastar_libs'] = cfg['Libs']
|
|
|
|
seastar_deps = 'practically_anything_can_change_so_lets_run_it_every_time_and_restat.'
|
|
|
|
args.user_cflags += " " + pkg_config("--cflags", "jsoncpp")
|
|
libs = ' '.join(['-lyaml-cpp', '-llz4', '-lz', '-lsnappy', pkg_config("--libs", "jsoncpp"),
|
|
maybe_static(args.staticboost, '-lboost_filesystem'), ' -lcrypt',
|
|
maybe_static(args.staticboost, '-lboost_date_time'),
|
|
])
|
|
|
|
if not args.staticboost:
|
|
args.user_cflags += ' -DBOOST_TEST_DYN_LINK'
|
|
|
|
for pkg in pkgs:
|
|
args.user_cflags += ' ' + pkg_config('--cflags', pkg)
|
|
libs += ' ' + pkg_config('--libs', pkg)
|
|
user_cflags = args.user_cflags
|
|
user_ldflags = args.user_ldflags
|
|
if args.staticcxx:
|
|
user_ldflags += " -static-libgcc -static-libstdc++"
|
|
if args.staticthrift:
|
|
thrift_libs = "-Wl,-Bstatic -lthrift -Wl,-Bdynamic"
|
|
else:
|
|
thrift_libs = "-lthrift"
|
|
|
|
outdir = 'build'
|
|
buildfile = 'build.ninja'
|
|
|
|
os.makedirs(outdir, exist_ok = True)
|
|
do_sanitize = True
|
|
if args.static:
|
|
do_sanitize = False
|
|
with open(buildfile, 'w') as f:
|
|
f.write(textwrap.dedent('''\
|
|
configure_args = {configure_args}
|
|
builddir = {outdir}
|
|
cxx = {cxx}
|
|
cxxflags = {user_cflags} {warnings} {defines}
|
|
ldflags = {user_ldflags}
|
|
libs = {libs}
|
|
pool link_pool
|
|
depth = {link_pool_depth}
|
|
pool seastar_pool
|
|
depth = 1
|
|
rule ragel
|
|
command = ragel -G2 -o $out $in
|
|
description = RAGEL $out
|
|
rule gen
|
|
command = echo -e $text > $out
|
|
description = GEN $out
|
|
rule swagger
|
|
command = seastar/json/json2code.py -f $in -o $out
|
|
description = SWAGGER $out
|
|
rule serializer
|
|
command = {python} ./idl-compiler.py --ns ser -f $in -o $out
|
|
description = IDL compiler $out
|
|
rule ninja
|
|
command = {ninja} -C $subdir $target
|
|
restat = 1
|
|
description = NINJA $out
|
|
rule copy
|
|
command = cp $in $out
|
|
description = COPY $out
|
|
''').format(**globals()))
|
|
for mode in build_modes:
|
|
modeval = modes[mode]
|
|
f.write(textwrap.dedent('''\
|
|
cxxflags_{mode} = -I. -I $builddir/{mode}/gen -I seastar -I seastar/build/{mode}/gen
|
|
rule cxx.{mode}
|
|
command = $cxx -MD -MT $out -MF $out.d {seastar_cflags} $cxxflags $cxxflags_{mode} -c -o $out $in
|
|
description = CXX $out
|
|
depfile = $out.d
|
|
rule link.{mode}
|
|
command = $cxx $cxxflags_{mode} {sanitize_libs} $ldflags {seastar_libs} -o $out $in $libs $libs_{mode}
|
|
description = LINK $out
|
|
pool = link_pool
|
|
rule link_stripped.{mode}
|
|
command = $cxx $cxxflags_{mode} -s {sanitize_libs} $ldflags {seastar_libs} -o $out $in $libs $libs_{mode}
|
|
description = LINK (stripped) $out
|
|
pool = link_pool
|
|
rule ar.{mode}
|
|
command = rm -f $out; ar cr $out $in; ranlib $out
|
|
description = AR $out
|
|
rule thrift.{mode}
|
|
command = thrift -gen cpp:cob_style -out $builddir/{mode}/gen $in
|
|
description = THRIFT $in
|
|
rule antlr3.{mode}
|
|
command = sed -e '/^#if 0/,/^#endif/d' $in > $builddir/{mode}/gen/$in && antlr3 $builddir/{mode}/gen/$in && sed -i 's/^\\( *\)\\(ImplTraits::CommonTokenType\\* [a-zA-Z0-9_]* = NULL;\\)$$/\\1const \\2/' build/{mode}/gen/${{stem}}Parser.cpp
|
|
description = ANTLR3 $in
|
|
''').format(mode = mode, **modeval))
|
|
f.write('build {mode}: phony {artifacts}\n'.format(mode = mode,
|
|
artifacts = str.join(' ', ('$builddir/' + mode + '/' + x for x in build_artifacts))))
|
|
compiles = {}
|
|
ragels = {}
|
|
swaggers = {}
|
|
serializers = {}
|
|
thrifts = set()
|
|
antlr3_grammars = set()
|
|
for binary in build_artifacts:
|
|
if binary in other:
|
|
continue
|
|
srcs = deps[binary]
|
|
objs = ['$builddir/' + mode + '/' + src.replace('.cc', '.o')
|
|
for src in srcs
|
|
if src.endswith('.cc')]
|
|
has_thrift = False
|
|
for dep in deps[binary]:
|
|
if isinstance(dep, Thrift):
|
|
has_thrift = True
|
|
objs += dep.objects('$builddir/' + mode + '/gen')
|
|
if isinstance(dep, Antlr3Grammar):
|
|
objs += dep.objects('$builddir/' + mode + '/gen')
|
|
if binary.endswith('.pc'):
|
|
vars = modeval.copy()
|
|
vars.update(globals())
|
|
pc = textwrap.dedent('''\
|
|
Name: Seastar
|
|
URL: http://seastar-project.org/
|
|
Description: Advanced C++ framework for high-performance server applications on modern hardware.
|
|
Version: 1.0
|
|
Libs: -L{srcdir}/{builddir} -Wl,--whole-archive -lseastar -Wl,--no-whole-archive {dbgflag} -Wl,--no-as-needed {static} {pie} -fvisibility=hidden -pthread {user_ldflags} {libs} {sanitize_libs}
|
|
Cflags: -std=gnu++1y {dbgflag} {fpie} -Wall -Werror -fvisibility=hidden -pthread -I{srcdir} -I{srcdir}/{builddir}/gen {user_cflags} {warnings} {defines} {sanitize} {opt}
|
|
''').format(builddir = 'build/' + mode, srcdir = os.getcwd(), **vars)
|
|
f.write('build $builddir/{}/{}: gen\n text = {}\n'.format(mode, binary, repr(pc)))
|
|
elif binary.endswith('.a'):
|
|
f.write('build $builddir/{}/{}: ar.{} {}\n'.format(mode, binary, mode, str.join(' ', objs)))
|
|
else:
|
|
if binary.startswith('tests/'):
|
|
local_libs = '$libs'
|
|
if binary not in tests_not_using_seastar_test_framework or binary in pure_boost_tests:
|
|
local_libs += ' ' + maybe_static(args.staticboost, '-lboost_unit_test_framework')
|
|
if has_thrift:
|
|
local_libs += ' ' + thrift_libs + ' ' + maybe_static(args.staticboost, '-lboost_system')
|
|
# Our code's debugging information is huge, and multiplied
|
|
# by many tests yields ridiculous amounts of disk space.
|
|
# So we strip the tests by default; The user can very
|
|
# quickly re-link the test unstripped by adding a "_g"
|
|
# to the test name, e.g., "ninja build/release/testname_g"
|
|
f.write('build $builddir/{}/{}: {}.{} {} {}\n'.format(mode, binary, tests_link_rule, mode, str.join(' ', objs),
|
|
'seastar/build/{}/libseastar.a'.format(mode)))
|
|
f.write(' libs = {}\n'.format(local_libs))
|
|
f.write('build $builddir/{}/{}_g: link.{} {} {}\n'.format(mode, binary, mode, str.join(' ', objs),
|
|
'seastar/build/{}/libseastar.a'.format(mode)))
|
|
f.write(' libs = {}\n'.format(local_libs))
|
|
else:
|
|
f.write('build $builddir/{}/{}: link.{} {} {}\n'.format(mode, binary, mode, str.join(' ', objs),
|
|
'seastar/build/{}/libseastar.a'.format(mode)))
|
|
if has_thrift:
|
|
f.write(' libs = {} {} $libs\n'.format(thrift_libs, maybe_static(args.staticboost, '-lboost_system')))
|
|
for src in srcs:
|
|
if src.endswith('.cc'):
|
|
obj = '$builddir/' + mode + '/' + src.replace('.cc', '.o')
|
|
compiles[obj] = src
|
|
elif src.endswith('.rl'):
|
|
hh = '$builddir/' + mode + '/gen/' + src.replace('.rl', '.hh')
|
|
ragels[hh] = src
|
|
elif src.endswith('.idl.hh'):
|
|
hh = '$builddir/' + mode + '/gen/' + src.replace('.idl.hh', '.dist.hh')
|
|
serializers[hh] = src
|
|
elif src.endswith('.json'):
|
|
hh = '$builddir/' + mode + '/gen/' + src + '.hh'
|
|
swaggers[hh] = src
|
|
elif src.endswith('.thrift'):
|
|
thrifts.add(src)
|
|
elif src.endswith('.g'):
|
|
antlr3_grammars.add(src)
|
|
else:
|
|
raise Exception('No rule for ' + src)
|
|
for obj in compiles:
|
|
src = compiles[obj]
|
|
gen_headers = list(ragels.keys())
|
|
gen_headers += ['seastar/build/{}/gen/http/request_parser.hh'.format(mode)]
|
|
gen_headers += ['seastar/build/{}/gen/http/http_response_parser.hh'.format(mode)]
|
|
for th in thrifts:
|
|
gen_headers += th.headers('$builddir/{}/gen'.format(mode))
|
|
for g in antlr3_grammars:
|
|
gen_headers += g.headers('$builddir/{}/gen'.format(mode))
|
|
gen_headers += list(swaggers.keys())
|
|
gen_headers += list(serializers.keys())
|
|
f.write('build {}: cxx.{} {} || {} \n'.format(obj, mode, src, ' '.join(gen_headers)))
|
|
if src in extra_cxxflags:
|
|
f.write(' cxxflags = {seastar_cflags} $cxxflags $cxxflags_{mode} {extra_cxxflags}\n'.format(mode = mode, extra_cxxflags = extra_cxxflags[src], **modeval))
|
|
for hh in ragels:
|
|
src = ragels[hh]
|
|
f.write('build {}: ragel {}\n'.format(hh, src))
|
|
for hh in swaggers:
|
|
src = swaggers[hh]
|
|
f.write('build {}: swagger {}\n'.format(hh,src))
|
|
for hh in serializers:
|
|
src = serializers[hh]
|
|
f.write('build {}: serializer {} | idl-compiler.py\n'.format(hh,src))
|
|
for thrift in thrifts:
|
|
outs = ' '.join(thrift.generated('$builddir/{}/gen'.format(mode)))
|
|
f.write('build {}: thrift.{} {}\n'.format(outs, mode, thrift.source))
|
|
for cc in thrift.sources('$builddir/{}/gen'.format(mode)):
|
|
obj = cc.replace('.cpp', '.o')
|
|
f.write('build {}: cxx.{} {}\n'.format(obj, mode, cc))
|
|
for grammar in antlr3_grammars:
|
|
outs = ' '.join(grammar.generated('$builddir/{}/gen'.format(mode)))
|
|
f.write('build {}: antlr3.{} {}\n stem = {}\n'.format(outs, mode, grammar.source,
|
|
grammar.source.rsplit('.', 1)[0]))
|
|
for cc in grammar.sources('$builddir/{}/gen'.format(mode)):
|
|
obj = cc.replace('.cpp', '.o')
|
|
f.write('build {}: cxx.{} {} || {}\n'.format(obj, mode, cc, ' '.join(serializers)))
|
|
f.write('build seastar/build/{mode}/libseastar.a seastar/build/{mode}/apps/iotune/iotune seastar/build/{mode}/gen/http/request_parser.hh seastar/build/{mode}/gen/http/http_response_parser.hh: ninja {seastar_deps}\n'
|
|
.format(**locals()))
|
|
f.write(' pool = seastar_pool\n')
|
|
f.write(' subdir = seastar\n')
|
|
f.write(' target = build/{mode}/libseastar.a build/{mode}/apps/iotune/iotune build/{mode}/gen/http/request_parser.hh build/{mode}/gen/http/http_response_parser.hh\n'.format(**locals()))
|
|
f.write(textwrap.dedent('''\
|
|
build build/{mode}/iotune: copy seastar/build/{mode}/apps/iotune/iotune
|
|
''').format(**locals()))
|
|
f.write('build {}: phony\n'.format(seastar_deps))
|
|
f.write(textwrap.dedent('''\
|
|
rule configure
|
|
command = {python} configure.py $configure_args
|
|
generator = 1
|
|
build build.ninja: configure | configure.py
|
|
rule cscope
|
|
command = find -name '*.[chS]' -o -name "*.cc" -o -name "*.hh" | cscope -bq -i-
|
|
description = CSCOPE
|
|
build cscope: cscope
|
|
rule clean
|
|
command = rm -rf build
|
|
description = CLEAN
|
|
build clean: clean
|
|
default {modes_list}
|
|
''').format(modes_list = ' '.join(build_modes), **globals()))
|