From f70aab2fbb461c32d317c83626aa707930d7e8a0 Mon Sep 17 00:00:00 2001 From: Vlad Zolotarov Date: Mon, 27 Jul 2015 10:22:08 +0300 Subject: [PATCH] locator: added ec2_multi_region_snitch This snitch in addition to what EC2Snitch does registers a reconnectable_snitch_helper that will make messenger_service connect to internal IPs when it connects to the nodes in the same data center with the current Node. Signed-off-by: Vlad Zolotarov New in v4: - Added dual license in newly added files. New in v3: - Returned the Apache license. New in v2: - Update the license to the latest version. ;) --- configure.py | 1 + locator/ec2_multi_region_snitch.cc | 132 +++++++++++++++++++++++++++++ locator/ec2_multi_region_snitch.hh | 56 ++++++++++++ 3 files changed, 189 insertions(+) create mode 100644 locator/ec2_multi_region_snitch.cc create mode 100644 locator/ec2_multi_region_snitch.hh diff --git a/configure.py b/configure.py index 502469db1b..2b657459ad 100755 --- a/configure.py +++ b/configure.py @@ -368,6 +368,7 @@ scylla_core = (['database.cc', 'locator/gossiping_property_file_snitch.cc', 'locator/production_snitch_base.cc', 'locator/ec2_snitch.cc', + 'locator/ec2_multi_region_snitch.cc', 'message/messaging_service.cc', 'service/migration_task.cc', 'service/storage_service.cc', diff --git a/locator/ec2_multi_region_snitch.cc b/locator/ec2_multi_region_snitch.cc new file mode 100644 index 0000000000..064c1d9c75 --- /dev/null +++ b/locator/ec2_multi_region_snitch.cc @@ -0,0 +1,132 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modified by Cloudius Systems. + * Copyright 2015 Cloudius Systems. + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#include "locator/ec2_multi_region_snitch.hh" +#include "locator/reconnectable_snitch_helper.hh" +#include "gms/gossiper.hh" +#include "service/storage_service.hh" + +namespace locator { +ec2_multi_region_snitch::ec2_multi_region_snitch(const sstring& fname, unsigned io_cpu_id) + : ec2_snitch(fname, io_cpu_id) {} + +future<> ec2_multi_region_snitch::start() { + _state = snitch_state::initializing; + + return ec2_snitch::load_config().then([this] { + if (engine().cpu_id() == io_cpu_id()) { + return aws_api_call(AWS_QUERY_SERVER_ADDR, PUBLIC_IP_QUERY_REQ).then([this](sstring pub_addr){ + inet_address local_public_address = inet_address(pub_addr); + logger().info("EC2MultiRegionSnitch using publicIP as identifier: {}", local_public_address); + + // + // Use the Public IP to broadcast Address to other nodes. + // + // Cassandra 2.1 manual explicitly instructs to set broadcast_address + // value to a public address in cassandra.yaml. + // + utils::fb_utilities::set_broadcast_address(local_public_address); + //DatabaseDescriptor.setBroadcastRpcAddress(local_public_address); + // + + return aws_api_call(AWS_QUERY_SERVER_ADDR, PRIVATE_IP_QUERY_REQ).then( + [this] (sstring priv_addr) { + _local_private_address = priv_addr; + }); + }).then([this] { + // + // Gossiper main instance is currently running on CPU0 - + // therefore we need to make sure the _local_private_address is + // set on the shard0 so that it may be used when Gossiper is + // going to invoke gossiper_starting() method. + // + return _my_distributed->invoke_on(0, + [this] (snitch_ptr& local_s) { + + if (engine().cpu_id() != io_cpu_id()) { + local_s->set_local_private_addr(_local_private_address); + } + }); + }).then([this] { + set_snitch_ready(); + }); + } + + set_snitch_ready(); + return make_ready_future<>(); + }); +} + +void ec2_multi_region_snitch::set_local_private_addr(const sstring& addr_str) { + _local_private_address = addr_str; +} + +void ec2_multi_region_snitch::gossiper_starting() { + // + // Note: currently gossiper "main" instance always runs on CPU0 therefore + // this function will be executed on CPU0 only. + // + ec2_snitch::gossiper_starting(); + + using namespace gms; + auto& g = get_local_gossiper(); + auto& ss = service::get_local_storage_service(); + + g.add_local_application_state(application_state::INTERNAL_IP, + ss.value_factory.internal_ip(_local_private_address)); + + if (!_helper_added) { + gms::get_local_gossiper().register_(new reconnectable_snitch_helper(_my_dc, true)); + _helper_added = true; + } +} + +using registry_2_params = class_registrator; +static registry_2_params registrator2("org.apache.cassandra.locator.EC2MultiRegionSnitch"); +static registry_2_params registrator2_short_name("EC2MultiRegionSnitch"); + + +using registry_1_param = class_registrator; +static registry_1_param registrator1("org.apache.cassandra.locator.EC2MultiRegionSnitch"); +static registry_1_param registrator1_short_name("EC2MultiRegionSnitch"); + +using registry_default = class_registrator; +static registry_default registrator_default("org.apache.cassandra.locator.EC2MultiRegionSnitch"); +static registry_default registrator_default_short_name("EC2MultiRegionSnitch"); + +} // namespace locator diff --git a/locator/ec2_multi_region_snitch.hh b/locator/ec2_multi_region_snitch.hh new file mode 100644 index 0000000000..de4e5557bd --- /dev/null +++ b/locator/ec2_multi_region_snitch.hh @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * Modified by Cloudius Systems. + * Copyright 2015 Cloudius Systems. + */ + +/* + * This file is part of Scylla. + * + * Scylla is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * Scylla is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with Scylla. If not, see . + */ + +#pragma once + +#include "locator/ec2_snitch.hh" + +namespace locator { +class ec2_multi_region_snitch : public ec2_snitch { +public: + ec2_multi_region_snitch(const sstring& fname = "", unsigned io_cpu_id = 0); + virtual void gossiper_starting() override; + virtual future<> start() override; + virtual void set_local_private_addr(const sstring& addr_str) override; +private: + static constexpr const char* PUBLIC_IP_QUERY_REQ = "/latest/meta-data/public-ipv4"; + static constexpr const char* PRIVATE_IP_QUERY_REQ = "/latest/meta-data/local-ipv4"; + sstring _local_private_address; + bool _helper_added = false; +}; +} // namespace locator