mirror of
https://github.com/scylladb/scylladb.git
synced 2026-05-12 19:02:12 +00:00
58fae5f4c1
We saw reactor stalls when closing SSTables. The backtrace looks like: Oct 12 19:00:51 dell-1 scylla[435045]: Backtrace:[Backtrace #0] void seastar::backtrace<seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}>(seastar::backtrace_buffer::append_backtrace()::{lambda(seastar::frame)#1}&&) at /home/sylla/scylla/seastar/util/backtrace.hh:56 seastar::backtrace_buffer::append_backtrace() at /home/sylla/scylla/seastar/core/reactor.cc:410 (inlined by) print_with_backtrace at /home/sylla/scylla/seastar/core/reactor.cc:431 seastar::reactor::block_notifier(int) at /home/sylla/scylla/seastar/core/reactor.cc:749 _L_unlock_13 at funlockfile.c:? std::experimental::fundamentals_v1::_Optional_base<range_bound<dht::ring_position>, true>::_Optional_base(std::experimental::fundamentals_v1::_Optional_base<range_bound<dht::ring_position>, true>&&) at /opt/scylladb/include/c++/7/experimental/optional:247 (inlined by) std::experimental::fundamentals_v1::optional<range_bound<dht::ring_position> >::optional(std::experimental::fundamentals_v1::optional<range_bound<dht::ring_position> >&&) at /opt/scylladb/include/c++/7/experimental/optional:493 (inlined by) wrapping_range<dht::ring_position>::wrapping_range(wrapping_range<dht::ring_position>&&) at /home/sylla/scylla/./range.hh:61 (inlined by) nonwrapping_range<dht::ring_position>::nonwrapping_range(nonwrapping_range<dht::ring_position>&&) at /home/sylla/scylla/./range.hh:430 (inlined by) void __gnu_cxx::new_allocator<nonwrapping_range<dht::ring_position> >::construct<nonwrapping_range<dht::ring_position>, nonwrapping_range<dht::ring_position> >(nonwrapping_range<dht::ring_position>*, nonwrapping_range<dht::ring_position>&&) at /opt/scylladb/include/c++/7/ext/new_allocator.h:136 (inlined by) void std::allocator_traits<std::allocator<nonwrapping_range<dht::ring_position> > >::construct<nonwrapping_range<dht::ring_position>, nonwrapping_range<dht::ring_position> >(std::allocator<nonwrapping_range<dht::ring_position> >&, nonwrapping_range<dht::ring_position>*, nonwrapping_range<dht::ring_position>&&) at /opt/scylladb/include/c++/7/bits/alloc_traits.h:475 (inlined by) nonwrapping_range<dht::ring_position>& std::deque<nonwrapping_range<dht::ring_position>, std::allocator<nonwrapping_range<dht::ring_position> > >::emplace_back<nonwrapping_range<dht::ring_position> >(nonwrapping_range<dht::ring_position>&&) at /opt/scylladb/include/c++/7/bits/deque.tcc:167 (inlined by) std::deque<nonwrapping_range<dht::ring_position>, std::allocator<nonwrapping_range<dht::ring_position> > >::push_back(nonwrapping_range<dht::ring_position>&&) at /opt/scylladb/include/c++/7/bits/stl_deque.h:1558 (inlined by) dht::split_range_to_single_shard(dht::i_partitioner const&, schema const&, nonwrapping_range<dht::ring_position> const&, unsigned int) at /home/sylla/scylla/dht/i_partitioner.cc:454 dht::split_range_to_single_shard(schema const&, nonwrapping_range<dht::ring_position> const&, unsigned int) at /home/sylla/scylla/dht/i_partitioner.cc:464 create_sharding_metadata at /home/sylla/scylla/sstables/sstables.cc:2075 (inlined by) sstables::sstable::write_scylla_metadata(seastar::io_priority_class const&, unsigned int, sstables::sstable_enabled_features) at /home/sylla/scylla/sstables/sstables.cc:2435 sstables::sstable_writer_m::consume_end_of_stream() at /home/sylla/scylla/sstables/sstables.cc:3483 sstables::compaction::finish_new_sstable(std::experimental::fundamentals_v1::optional<sstables::sstable_writer>&, seastar::lw_shared_ptr<sstables::sstable>&) at /home/sylla/scylla/sstables/compaction.cc:338 (inlined by) sstables::regular_compaction::stop_sstable_writer() at /home/sylla/scylla/sstables/compaction.cc:579 (inlined by) sstables::regular_compaction::finish_sstable_writer() at /home/sylla/scylla/sstables/compaction.cc:585 sstables::compacting_sstable_writer::consume_end_of_stream() at /home/sylla/scylla/sstables/compaction.cc:494 (inlined by) auto compact_mutation_state<(emit_only_live_rows)0, (compact_for_sstables)1>::consume_end_of_stream<sstables::compacting_sstable_writer>(sstables::compacting_sstable_writer&) at /home/sylla/scylla/./mutation_compactor.hh:292 (inlined by) compact_mutation<(emit_only_live_rows)0, (compact_for_sstables)1, sstables::compacting_sstable_writer>::consume_end_of_stream() at /home/sylla/scylla/./mutation_compactor.hh:397 (inlined by) stable_flattened_mutations_consumer<compact_for_compaction<sstables::compacting_sstable_writer> >::consume_end_of_stream() at /home/sylla/scylla/./mutation_reader.hh:366 (inlined by) auto flat_mutation_reader::impl::consume_in_thread<stable_flattened_mutations_consumer<compact_for_compaction<sstables::compacting_sstable_writer> >, std::function<bool (dht::decorated_key const&)> >(stable_flattened_mutations_consumer<compact_for_compaction<sstables::compacting_sstable_writer> >, std::function<bool (dht::decorated_key const&)>, std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >) at /home/sylla/scylla/./flat_mutation_reader.hh:288 (inlined by) auto flat_mutation_reader::consume_in_thread<stable_flattened_mutations_consumer<compact_for_compaction<sstables::compacting_sstable_writer> >, std::function<bool (dht::decorated_key const&)> >(stable_flattened_mutations_consumer<compact_for_compaction<sstables::compacting_sstable_writer> >, std::function<bool (dht::decorated_key const&)>, std::chrono::time_point<seastar::lowres_clock, std::chrono::duration<long, std::ratio<1l, 1000l> > >) at /home/sylla/scylla/./flat_mutation_reader.hh:370 (inlined by) operator() at /home/sylla/scylla/sstables/compaction.cc:757 (inlined by) apply at /home/sylla/scylla/seastar/core/apply.hh:35 (inlined by) apply<sstables::compaction::run(std::unique_ptr<sstables::compaction>)::<lambda()> > at /home/sylla/scylla/seastar/core/apply.hh:43 (inlined by) apply<sstables::compaction::run(std::unique_ptr<sstables::compaction>)::<lambda()> > at /home/sylla/scylla/seastar/core/future.hh:1309 (inlined by) operator() at /home/sylla/scylla/./seastar/core/thread.hh:315 (inlined by) _M_invoke at /opt/scylladb/include/c++/7/bits/std_function.h:316 std::function<void ()>::operator()() const at /opt/scylladb/include/c++/7/bits/std_function.h:706 (inlined by) seastar::thread_context::main() at /home/sylla/scylla/seastar/core/thread.cc:313 The call chain is: sstable_writer_k_l::consume_end_of_stream and mc::writer::consume_end_of_stream -> sstable::write_scylla_metadata -> create_sharding_metadata -> dht::split_range_to_single_shard Since sstable writer assumes a thread context. We can futurize dht::split_range_to_single_shard. Fixes: #3846 Tests: dtest + build/dev/tests/partitioner_test