Merge "Fixes for AMI" from Shlomi

"The patch fixes a few issues caused by generalizing the ami scripts. The
 scylla_bootparam_setup requires invocation with ami flag. The
 scylla_install is missing some steps executed by the scylla-ami.sh."

SCYLLA-VERSION-GEN

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-VERSION=development
+VERSION=0.14
 
 if test -f version
 then

conf/scylla.yaml

@@ -824,3 +824,17 @@ commitlog_total_space_in_mb: -1
 # reducing overhead from the TCP protocol itself, at the cost of increasing
 # latency if you block for cross-datacenter responses.
 # inter_dc_tcp_nodelay: false
+
+# Relaxation of environment checks.
+#
+# Scylla places certain requirements on its environment.  If these requirements are
+# not met, performance and reliability can be degraded.
+#
+# These requirements include:
+#    - A filesystem with good support for aysnchronous I/O (AIO). Currently,
+#      this means XFS.
+#
+# false: strict environment checks are in place; do not start if they are not met.
+# true: relaxed environment checks; performance and reliability may degraade.
+#
+# developer_mode: false

db/config.hh

@@ -724,7 +724,8 @@ public:
     val(replace_address, sstring, "", Used, "The listen_address or broadcast_address of the dead node to replace. Same as -Dcassandra.replace_address.") \
     val(replace_address_first_boot, sstring, "", Used, "Like replace_address option, but if the node has been bootstrapped sucessfully it will be ignored. Same as -Dcassandra.replace_address_first_boot.") \
     val(override_decommission, bool, false, Used, "Set true to force a decommissioned node to join the cluster") \
-    val(ring_delay_ms, uint32_t, 30 * 1000, Used, "Time a node waits to hear from other nodes before joining the ring in milliseconds. Same as -Dcassandra.ring_delay_ms in cassandra.")
+    val(ring_delay_ms, uint32_t, 30 * 1000, Used, "Time a node waits to hear from other nodes before joining the ring in milliseconds. Same as -Dcassandra.ring_delay_ms in cassandra.") \
+    val(developer_mode, bool, false, Used, "Relax environement checks. Setting to true can reduce performance and reliability significantly.") \
     /* done! */
 
 #define _make_value_member(name, type, deflt, status, desc, ...)    \

dist/common/limits.d/scylla.conf

@@ -1,5 +1,5 @@
 scylla  -  core     unlimited
 scylla  -  memlock  unlimited
-scylla  -  nofile   100000
+scylla  -  nofile   200000
 scylla  -  as       unlimited
 scylla  -  nproc    8096

dist/common/scripts/scylla_bootparam_setup

@@ -20,14 +20,14 @@ while getopts a OPT; do
     esac
 done
 
-
 . /etc/os-release
-. /etc/sysconfig/scylla-server
 
 if [ $AMI -eq 1 ]; then
+    . /etc/sysconfig/scylla-server
     sed -e "s#append #append clocksource=tsc tsc=reliable hugepagesz=2M hugepages=$NR_HUGEPAGES #" /boot/extlinux/extlinux.conf > /tmp/extlinux.conf
     mv /tmp/extlinux.conf /boot/extlinux/extlinux.conf
 else
+    . /etc/sysconfig/scylla-server
     if [ ! -f /etc/default/grub ]; then
         echo "Unsupported bootloader"
         exit 1

dist/common/scripts/scylla_save_coredump

@@ -6,4 +6,5 @@ FILE=$1
 TIME=`date --date @$2 +%F-%T`
 PID=$3
 
+mkdir -p /var/lib/scylla/coredump
 /usr/bin/gzip -c > /var/lib/scylla/coredump/core.$FILE-$TIME-$PID.gz

dist/docker/start-scylla

@@ -4,6 +4,7 @@ IP=$(hostname -i)
 sed -e "s/seeds:.*/seeds: $IP/g" /var/lib/scylla/conf/scylla.yaml > $HOME/scylla.yaml
 /usr/bin/scylla --log-to-syslog 1 \
                 --log-to-stdout 0 \
+                --developer-mode true \
                 --default-log-level info \
                 --options-file $HOME/scylla.yaml \
                 --listen-address $IP \

dist/redhat/scylla-server.spec.in

@@ -51,6 +51,7 @@ install -m644 conf/scylla.yaml $RPM_BUILD_ROOT%{_sysconfdir}/scylla/
 install -m644 conf/cassandra-rackdc.properties $RPM_BUILD_ROOT%{_sysconfdir}/scylla/
 install -m644 dist/redhat/systemd/scylla-server.service $RPM_BUILD_ROOT%{_unitdir}/
 install -m755 dist/common/scripts/* $RPM_BUILD_ROOT%{_prefix}/lib/scylla/
+install -m755 dist/redhat/scripts/* $RPM_BUILD_ROOT%{_prefix}/lib/scylla/
 install -m755 seastar/scripts/posix_net_conf.sh  $RPM_BUILD_ROOT%{_prefix}/lib/scylla/
 install -m755 seastar/dpdk/tools/dpdk_nic_bind.py $RPM_BUILD_ROOT%{_prefix}/lib/scylla/
 install -m755 build/release/scylla $RPM_BUILD_ROOT%{_bindir}

dist/redhat/systemd/scylla-server.service

@@ -5,7 +5,7 @@ After=network.target libvirtd.service
 [Service]
 Type=simple
 LimitMEMLOCK=infinity
-LimitNOFILE=100000
+LimitNOFILE=200000
 LimitAS=infinity
 LimitNPROC=8096
 EnvironmentFile=/etc/sysconfig/scylla-server

dist/ubuntu/debian/rules

@@ -38,6 +38,7 @@ override_dh_auto_install:
 	mkdir -p $(SCRIPTS) && \
 	cp $(CURDIR)/seastar/dpdk/tools/dpdk_nic_bind.py $(SCRIPTS)
 	cp $(CURDIR)/dist/common/scripts/* $(SCRIPTS)
+	cp $(CURDIR)/dist/ubuntu/scripts/* $(SCRIPTS)
 
 	mkdir -p $(SWAGGER) && \
 	cp -r $(CURDIR)/swagger-ui/dist $(SWAGGER)

dist/ubuntu/scripts/scylla_run

@@ -0,0 +1,19 @@
+#!/bin/bash -e
+
+args="--log-to-syslog 1 --log-to-stdout 0 --default-log-level info $SCYLLA_ARGS"
+
+if [ "$NETWORK_MODE" = "posix" ]; then
+    args="$args --network-stack posix"
+elif [ "$NETWORK_MODE" = "virtio" ]; then
+    args="$args --network-stack native"
+elif [ "$NETWORK_MODE" = "dpdk" ]; then
+    args="$args --network-stack native --dpdk-pmd"
+fi
+
+export HOME=/var/lib/scylla
+ulimit -c unlimited
+ulimit -l unlimited
+ulimit -n 200000
+ulimit -m unlimited
+ulimit -u 8096
+exec sudo -E -u $USER /usr/bin/scylla $args

main.cc

@@ -45,6 +45,8 @@
 #include "release.hh"
 #include <cstdio>
 #include <core/file.hh>
+#include <sys/time.h>
+#include <sys/resource.h>
 
 logging::logger startlog("init");
 
@@ -100,12 +102,19 @@ static logging::log_level to_loglevel(sstring level) {
     }
 }
 
-static future<> disk_sanity(sstring path) {
-    return check_direct_io_support(path).then([path] {
-        return file_system_at(path).then([path] (auto fs) {
+static future<> disk_sanity(sstring path, bool developer_mode) {
+    return check_direct_io_support(path).then([path, developer_mode] {
+        return file_system_at(path).then([path, developer_mode] (auto fs) {
             if (fs != fs_type::xfs) {
-                startlog.warn("{} is not on XFS. This is a non-supported setup, and performance is expected to be very bad.\n"
-                    "For better performance, placing your data on XFS-formatted directories is strongly recommended", path);
+                if (!developer_mode) {
+                    startlog.error("{} is not on XFS. This is a non-supported setup, and performance is expected to be very bad.\n"
+                            "For better performance, placing your data on XFS-formatted directories is required."
+                            " To override this error, see the developer_mode configuration option.", path);
+                    throw std::runtime_error(sprint("invalid configuration: path \"%s\" on unsupported filesystem", path));
+                } else {
+                    startlog.warn("{} is not on XFS. This is a non-supported setup, and performance is expected to be very bad.\n"
+                            "For better performance, placing your data on XFS-formatted directories is strongly recommended", path);
+                }
             }
         });
     });
@@ -171,6 +180,28 @@ private:
 
 class bad_configuration_error : public std::exception {};
 
+static
+void
+verify_rlimit(bool developer_mode) {
+    struct rlimit lim;
+    int r = getrlimit(RLIMIT_NOFILE, &lim);
+    if (r == -1) {
+        throw std::system_error(errno, std::system_category());
+    }
+    auto recommended = 200'000U;
+    auto min = 10'000U;
+    if (lim.rlim_cur < min) {
+        if (developer_mode) {
+            startlog.warn("NOFILE rlimit too low (recommended setting {}, minimum setting {};"
+                          " you may run out of file descriptors.", recommended, min);
+        } else {
+            startlog.error("NOFILE rlimit too low (recommended setting {}, minimum setting {};"
+                          " refusing to start.", recommended, min);
+            throw std::runtime_error("NOFILE rlimit too low");
+        }
+    }
+}
+
 int main(int ac, char** av) {
     runtime::init_uptime();
     std::setvbuf(stdout, nullptr, _IOLBF, 1000);
@@ -210,6 +241,7 @@ int main(int ac, char** av) {
         return read_config(opts, *cfg).then([cfg, &db, &qp, &proxy, &mm, &ctx, &opts, &dirs]() {
             apply_logger_settings(cfg->default_log_level(), cfg->logger_log_level(),
                     cfg->log_to_stdout(), cfg->log_to_syslog());
+            verify_rlimit(cfg->developer_mode());
             dht::set_global_partitioner(cfg->partitioner());
             auto start_thrift = cfg->start_rpc();
             uint16_t api_port = cfg->api_port();
@@ -347,9 +379,9 @@ int main(int ac, char** av) {
                 directories.insert(db.local().get_config().data_file_directories().cbegin(),
                         db.local().get_config().data_file_directories().cend());
                 directories.insert(db.local().get_config().commitlog_directory());
-                return do_with(std::move(directories), [] (auto& directories) {
-                    return parallel_for_each(directories, [] (sstring pathname) {
-                        return disk_sanity(pathname);
+                return do_with(std::move(directories), [&db] (auto& directories) {
+                    return parallel_for_each(directories, [&db] (sstring pathname) {
+                        return disk_sanity(pathname, db.local().get_config().developer_mode());
                     });
                 });
             }).then([&db] {

scripts/scylla_install

@@ -83,14 +83,18 @@ else
         fi
         grep -v ' - mounts' /etc/cloud/cloud.cfg > /tmp/cloud.cfg
         mv /tmp/cloud.cfg /etc/cloud/cloud.cfg
+        mv /home/fedora/scylla-ami /usr/lib/scylla/scylla-ami
+        chmod a+rx /usr/lib/scylla/scylla-ami/ds2_configure.py
     fi
     systemctl enable scylla-server.service
     systemctl enable scylla-jmx.service
     /usr/lib/scylla/scylla_ntp_setup -a
 fi
-/usr/lib/scylla/scylla_bootparam_setup
 /usr/lib/scylla/scylla_coredump_setup
 if [ $AMI -eq 0 ]; then
+    /usr/lib/scylla/scylla_bootparam_setup
     /usr/lib/scylla/scylla_raid_setup -d $DISKS -u
+else
+    /usr/lib/scylla/scylla_bootparam_setup -a
 fi
 /usr/lib/scylla/scylla_sysconfig_setup $SYSCONFIG_SETUP_ARGS