dist: remove rpm dependency to libvirt

This is for testing virtio mode, since we don't officially recommend to use virtio mode we should drop it. Signed-off-by: Takuya ASADA <syuu@cloudius-systems.com>
ami: Improve scylla raid setup
2015-09-24 12:02:36 +02:00 · 2015-09-24 01:20:04 -07:00 · 2015-09-24 00:41:00 -07:00 · 2015-09-23 23:38:42 -07:00 · 2015-09-23 21:59:52 -07:00 · 2015-09-23 20:39:02 -07:00
10 changed files with 139 additions and 100 deletions
--- a/database.cc
+++ b/database.cc
@@ -811,7 +811,7 @@ future<> database::populate_keyspace(sstring datadir, sstring ks_name) {
    if (i == _keyspaces.end()) {
        dblog.warn("Skipping undefined keyspace: {}", ks_name);
    } else {
-        dblog.warn("Populating Keyspace {}", ks_name);
+        dblog.info("Populating Keyspace {}", ks_name);
        return lister::scan_dir(ksdir, directory_entry_type::directory, [this, ksdir, ks_name] (directory_entry de) {
            auto comps = parse_fname(de.name);
            if (comps.size() < 2) {
--- a/dist/ami/build_ami.sh
+++ b/dist/ami/build_ami.sh
@@ -1,9 +1,7 @@
 #!/bin/sh -e

-if [ ! -e dist/ami/build_ami.sh ] || [ ! -e ../scylla-jmx/dist/redhat/build_rpm.sh ] || [ ! -e ../cassandra/dist/redhat/build_rpm.sh ]; then
+if [ ! -e dist/ami/build_ami.sh ]; then
    echo "run build_ami.sh in top of scylla dir"
-    echo "please make sure scylla-jmx is checked out under the same directory as scylla"
-    echo "please make sure cassandra with scylla tools branch checked out under the same directory as scylla"
    exit 1
 fi

@@ -14,39 +12,6 @@ if [ ! -f variables.json ]; then
    exit 1
 fi

-if [ ! -f files/scylla-server.rpm ] || [ ! -f files/scylla-server-debuginfo.rpm ]; then
-    cd ../../
-    dist/redhat/build_rpm.sh
-    SCYLLA_VERSION=$(cat build/SCYLLA-VERSION-FILE)
-    SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE)
-    RPM=`ls build/rpms/scylla-server-$SCYLLA_VERSION-$SCYLLA_RELEASE*.x86_64.rpm|grep -v debuginfo`
-    cp $RPM dist/ami/files/scylla-server.rpm
-    cp build/rpms/scylla-server-debuginfo-$SCYLLA_VERSION-$SCYLLA_RELEASE*.x86_64.rpm dist/ami/files/scylla-server-debuginfo.rpm
-    cd -
-fi
-
-if [ ! -f files/scylla-jmx.rpm ]; then
-    CWD=`pwd`
-    cd ../../../scylla-jmx
-    dist/redhat/build_rpm.sh
-    SCYLLA_VERSION=$(cat build/SCYLLA-VERSION-FILE)
-    SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE)
-    RPM=`ls build/rpms/scylla-jmx-$SCYLLA_VERSION-$SCYLLA_RELEASE*.noarch.rpm`
-    cp $RPM $CWD/files/scylla-jmx.rpm
-    cd -
-fi
-
-if [ ! -f files/scylla-tools.rpm ]; then
-    CWD=`pwd`
-    cd ../../../cassandra
-    dist/redhat/build_rpm.sh
-    SCYLLA_VERSION=$(cat build/SCYLLA-VERSION-FILE)
-    SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE)
-    RPM=`ls build/rpms/scylla-tools-$SCYLLA_VERSION-$SCYLLA_RELEASE*.noarch.rpm`
-    cp $RPM $CWD/files/scylla-tools.rpm
-    cd -
-fi
-
 if [ ! -d packer ]; then
    wget https://dl.bintray.com/mitchellh/packer/packer_0.8.6_linux_amd64.zip
    mkdir packer
--- a/dist/ami/files/scylla-ami
+++ b/dist/ami/files/scylla-ami
--- a/dist/ami/files/scylla-setup.sh
+++ b/dist/ami/files/scylla-setup.sh
@@ -1,27 +1,49 @@
 #!/bin/sh -e

-if [ -f /dev/md0 ]; then
+if [ -b /dev/md0 ]; then
    echo "RAID already constructed."
    exit 1
 fi
-mdadm --create --verbose --force --run /dev/md0 --level=0 -c256 --raid-devices=2 /dev/xvdb /dev/xvdc
-blockdev --setra 65536 /dev/md0
-mkfs.xfs /dev/md0 -f
-echo "DEVICE /dev/xvdb /dev/xvdc" > /etc/mdadm.conf
-mdadm --detail --scan >> /etc/mdadm.conf 
-UUID=`blkid /dev/md0 | awk '{print $2}'`
-mkdir /data
-echo "$UUID /data xfs noatime 0 0" >> /etc/fstab
-mount /data
+
+dnf update -y
+
+DISKS=""
+NR=0
+for i in xvd{b..z}; do
+    if [ -b /dev/$i ];then
+        echo Found disk /dev/$i
+        DISKS="$DISKS /dev/$i"
+        NR=$((NR+1))
+    fi
+done
+
+echo Creating RAID0 for scylla using $NR disk\(s\): $DISKS
+
+if [ $NR -ge 1 ]; then
+    mdadm --create --verbose --force --run /dev/md0 --level=0 -c256 --raid-devices=$NR $DISKS
+    blockdev --setra 65536 /dev/md0
+    mkfs.xfs /dev/md0 -f
+    echo "DEVICE $DISKS" > /etc/mdadm.conf
+    mdadm --detail --scan >> /etc/mdadm.conf
+    UUID=`blkid /dev/md0 | awk '{print $2}'`
+    mkdir /data
+    echo "$UUID /data xfs noatime 0 0" >> /etc/fstab
+    mount /data
+else
+   echo "WARN: Scylla is not using XFS to store data. Perforamnce will suffer." > /home/fedora/WARN_PLEASE_READ.TXT
+fi
+
 mkdir -p /data/data
 mkdir -p /data/commitlog
 chown scylla:scylla /data/*
+
 CPU_NR=`cat /proc/cpuinfo |grep processor|wc -l`
 if [ $CPU_NR -ge 8 ]; then
    NR=$((CPU_NR - 1))
    echo SCYLLA_ARGS=\"--cpuset 1-$NR  --smp $NR\" >> /etc/sysconfig/scylla-server
    echo SET_NIC=\"yes\" >> /etc/sysconfig/scylla-server
 fi
+
 /usr/lib/scylla/scylla-ami/ds2_configure.py
 systemctl disable scylla-setup.service
 systemctl enable scylla-server.service
--- a/dist/ami/files/scylla.repo
+++ b/dist/ami/files/scylla.repo
@@ -0,0 +1,11 @@
+[scylla]
+name=Scylla for Fedora $releasever - $basearch
+baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/fedora/$releasever/$basearch/
+enabled=1
+gpgcheck=0
+
+[scylla-generic]
+name=Scylla for Fedora $releasever 
+baseurl=https://s3.amazonaws.com/downloads.scylladb.com/rpm/fedora/$releasever/noarch/
+enabled=1
+gpgcheck=0
--- a/dist/ami/files/setup-ami.sh
+++ b/dist/ami/files/setup-ami.sh
@@ -0,0 +1,20 @@
+#!/bin/sh -e
+
+setenforce 0
+sed -e "s/enforcing/disabled/" /etc/sysconfig/selinux > /tmp/selinux
+mv /tmp/selinux /etc/sysconfig/
+dnf update -y
+mv /home/fedora/scylla.repo /etc/yum.repos.d/
+dnf install -y scylla-server scylla-server-debuginfo scylla-jmx scylla-tools
+dnf install -y mdadm xfsprogs
+cp /home/fedora/coredump.conf /etc/systemd/coredump.conf
+mv /home/fedora/scylla-setup.service /usr/lib/systemd/system
+mv /home/fedora/scylla-setup.sh /usr/lib/scylla
+chmod a+rx /usr/lib/scylla/scylla-setup.sh
+mv /home/fedora/scylla-ami /usr/lib/scylla/scylla-ami
+chmod a+rx /usr/lib/scylla/scylla-ami/ds2_configure.py
+systemctl enable scylla-setup.service
+sed -e 's!/var/lib/scylla/data!/data/data!' -e 's!commitlog_directory: /var/lib/scylla/commitlog!commitlog_directory: /data/commitlog!' /var/lib/scylla/conf/scylla.yaml > /tmp/scylla.yaml
+mv /tmp/scylla.yaml /var/lib/scylla/conf
+grep -v ' - mounts' /etc/cloud/cloud.cfg > /tmp/cloud.cfg
+mv /tmp/cloud.cfg /etc/cloud/cloud.cfg
--- a/dist/ami/scylla.json
+++ b/dist/ami/scylla.json
@@ -24,20 +24,7 @@
    {
      "type": "shell",
      "inline": [
-        "sudo dnf update -y",
-        "sudo dnf install -y /home/fedora/*.rpm",
-        "sudo dnf install -y mdadm xfsprogs",
-        "sudo cp /home/fedora/coredump.conf /etc/systemd/coredump.conf",
-        "sudo mv /home/fedora/scylla-setup.service /usr/lib/systemd/system",
-        "sudo mv /home/fedora/scylla-setup.sh /usr/lib/scylla",
-        "sudo chmod a+rx /usr/lib/scylla/scylla-setup.sh",
-	"sudo mv /home/fedora/scylla-ami /usr/lib/scylla/scylla-ami",
-	"sudo chmod a+rx /usr/lib/scylla/scylla-ami/ds2_configure.py",
-        "sudo systemctl enable scylla-setup.service",
-	"sudo sed -e 's!/var/lib/scylla/data!/data/data!' -e 's!commitlog_directory: /var/lib/scylla/commitlog!commitlog_directory: /data/commitlog!' /var/lib/scylla/conf/scylla.yaml > /tmp/scylla.yaml",
-        "sudo mv /tmp/scylla.yaml /var/lib/scylla/conf",
-	"grep -v ' - mounts' /etc/cloud/cloud.cfg > /tmp/cloud.cfg",
-	"sudo mv /tmp/cloud.cfg /etc/cloud/cloud.cfg"
+	"sudo sh -x -e /home/fedora/setup-ami.sh"
      ]
    }
  ],
--- a/dist/redhat/build_rpm.sh
+++ b/dist/redhat/build_rpm.sh
@@ -37,6 +37,6 @@ if [ "$OS" = "Fedora" ]; then
    rpmbuild -bs --define "_topdir $RPMBUILD" $RPMBUILD/SPECS/scylla-server.spec
    mock rebuild --resultdir=`pwd`/build/rpms $RPMBUILD/SRPMS/scylla-server-$VERSION*.src.rpm
 else
-    sudo yum-builddep $RPMBUILD/SPECS/scylla-server.spec
+    sudo yum-builddep -y  $RPMBUILD/SPECS/scylla-server.spec
    rpmbuild -ba --define "_topdir $RPMBUILD" $RPMBUILD/SPECS/scylla-server.spec
 fi
--- a/dist/redhat/centos_dep/build_dependency.sh
+++ b/dist/redhat/centos_dep/build_dependency.sh
@@ -1,13 +1,36 @@
 #!/bin/sh -e
 export RPMBUILD=`pwd`/build/rpmbuild

+do_install()
+{
+    pkg=$1
+    name=${pkg/%.rpm/}
+    if ! rpm -qs $name  >/dev/null 2>&1; then
+	sudo yum install -y $RPMBUILD/RPMS/x86_64/$pkg || sudo yum install -y $RPMBUILD/RPMS/noarch/$pkg
+    fi
+    echo Install $name done
+}
+
 sudo yum install -y wget yum-utils rpm-build rpmdevtools gcc gcc-c++ make patch
 mkdir -p build/srpms
 cd build/srpms
-wget http://download.fedoraproject.org/pub/fedora/linux/releases/22/Everything/source/SRPMS/b/boost-1.57.0-6.fc22.src.rpm
-wget http://download.fedoraproject.org/pub/fedora/linux/releases/22/Everything/source/SRPMS/n/ninja-build-1.5.3-2.fc22.src.rpm
-wget http://download.fedoraproject.org/pub/fedora/linux/releases/22/Everything/source/SRPMS/r/ragel-6.8-3.fc22.src.rpm
-wget http://download.fedoraproject.org/pub/fedora/linux/releases/22/Everything/source/SRPMS/r/re2c-0.13.5-9.fc22.src.rpm
+
+if [ ! -f boost-1.57.0-6.fc22.src.rpm ]; then
+    wget http://download.fedoraproject.org/pub/fedora/linux/releases/22/Everything/source/SRPMS/b/boost-1.57.0-6.fc22.src.rpm
+fi
+
+if [ ! -f ninja-build-1.5.3-2.fc22.src.rpm ]; then
+    wget http://download.fedoraproject.org/pub/fedora/linux/releases/22/Everything/source/SRPMS/n/ninja-build-1.5.3-2.fc22.src.rpm
+fi
+
+if [ ! -f ragel-6.8-3.fc22.src.rpm ]; then
+   wget http://download.fedoraproject.org/pub/fedora/linux/releases/22/Everything/source/SRPMS/r/ragel-6.8-3.fc22.src.rpm
+fi
+
+if [ ! -f re2c-0.13.5-9.fc22.src.rpm ]; then
+   wget http://download.fedoraproject.org/pub/fedora/linux/releases/22/Everything/source/SRPMS/r/re2c-0.13.5-9.fc22.src.rpm
+fi
+
 cd -

 sudo yum install -y epel-release
@@ -15,41 +38,55 @@ sudo yum install -y cryptopp cryptopp-devel jsoncpp jsoncpp-devel lz4 lz4-devel
 sudo ln -sf /usr/bin/python3.4 /usr/bin/python3

 sudo yum install -y scl-utils
-sudo yum install -y https://www.softwarecollections.org/en/scls/rhscl/devtoolset-3/epel-7-x86_64/download/rhscl-devtoolset-3-epel-7-x86_64.noarch.rpm
+if ! rpm  -qs rhscl-devtoolset-3-epel-7-x86_64-1-2.noarch; then
+    sudo yum install -y https://www.softwarecollections.org/en/scls/rhscl/devtoolset-3/epel-7-x86_64/download/rhscl-devtoolset-3-epel-7-x86_64.noarch.rpm
+fi
 sudo yum install -y devtoolset-3-gcc-c++

 sudo yum install -y python-devel libicu-devel openmpi-devel mpich-devel libstdc++-devel bzip2-devel zlib-devel
-rpmbuild --define "_topdir $RPMBUILD" --without python3 --rebuild build/srpms/boost-1.57.0-6.fc22.src.rpm
-sudo yum install -y `ls $RPMBUILD/RPMS/x86_64/boost*|grep -v debuginfo`
+if [ ! -f $RPMBUILD/RPMS/x86_64/boost-1.57.0-6.el7.centos.x86_64.rpm ]; then
+    rpmbuild --define "_topdir $RPMBUILD" --without python3 --rebuild build/srpms/boost-1.57.0-6.fc22.src.rpm
+fi
+for i in `ls $RPMBUILD/RPMS/x86_64/boost*|grep -v debuginfo`;do
+    do_install `basename $i`
+done

-rpmbuild --define "_topdir $RPMBUILD" --rebuild build/srpms/re2c-0.13.5-9.fc22.src.rpm
-sudo yum install -y $RPMBUILD/RPMS/x86_64/re2c-0.13.5-9.el7.centos.x86_64.rpm
+if [ ! -f $RPMBUILD/RPMS/x86_64/re2c-0.13.5-9.el7.centos.x86_64.rpm ]; then
+    rpmbuild --define "_topdir $RPMBUILD" --rebuild build/srpms/re2c-0.13.5-9.fc22.src.rpm
+fi
+do_install re2c-0.13.5-9.el7.centos.x86_64.rpm

-rpm --define "_topdir $RPMBUILD" -ivh build/srpms/ninja-build-1.5.3-2.fc22.src.rpm
-patch $RPMBUILD/SPECS/ninja-build.spec < dist/redhat/centos_dep/ninja-build.diff
-rpmbuild --define "_topdir $RPMBUILD" -ba $RPMBUILD/SPECS/ninja-build.spec
-sudo yum install -y $RPMBUILD/RPMS/x86_64/ninja-build-1.5.3-2.el7.centos.x86_64.rpm
+if [ ! -f $RPMBUILD/RPMS/x86_64/ninja-build-1.5.3-2.el7.centos.x86_64.rpm ]; then
+   rpm --define "_topdir $RPMBUILD" -ivh build/srpms/ninja-build-1.5.3-2.fc22.src.rpm
+   patch $RPMBUILD/SPECS/ninja-build.spec < dist/redhat/centos_dep/ninja-build.diff
+   rpmbuild --define "_topdir $RPMBUILD" -ba $RPMBUILD/SPECS/ninja-build.spec
+fi
+do_install ninja-build-1.5.3-2.el7.centos.x86_64.rpm

-sudo yum install -y gcc-objc
-rpm --define "_topdir $RPMBUILD" -ivh build/srpms/ragel-6.8-3.fc22.src.rpm
-patch $RPMBUILD/SPECS/ragel.spec < dist/redhat/centos_dep/ragel.diff
-rpmbuild --define "_topdir $RPMBUILD" -ba $RPMBUILD/SPECS/ragel.spec
-sudo yum install -y $RPMBUILD/RPMS/x86_64/ragel-6.8-3.el7.centos.x86_64.rpm
-
-mkdir build/antlr3-tool-3.5.2
-cp dist/redhat/centos_dep/antlr3 build/antlr3-tool-3.5.2
-cd build/antlr3-tool-3.5.2
-wget http://www.antlr3.org/download/antlr-3.5.2-complete-no-st3.jar
-cd -
-cd build
-tar cJpf $RPMBUILD/SOURCES/antlr3-tool-3.5.2.tar.xz antlr3-tool-3.5.2
-cd -
-rpmbuild --define "_topdir $RPMBUILD" -ba dist/redhat/centos_dep/antlr3-tool.spec
-sudo yum install -y $RPMBUILD/RPMS/noarch/antlr3-tool-3.5.2-1.el7.centos.noarch.rpm
-
-wget -O build/3.5.2.tar.gz https://github.com/antlr/antlr3/archive/3.5.2.tar.gz
-mv build/3.5.2.tar.gz $RPMBUILD/SOURCES
-rpmbuild --define "_topdir $RPMBUILD" -ba dist/redhat/centos_dep/antlr3-C++-devel.spec
-sudo yum install -y $RPMBUILD/RPMS/x86_64/antlr3-C++-devel-3.5.2-1.el7.centos.x86_64.rpm
+if [ ! -f $RPMBUILD/RPMS/x86_64/ragel-6.8-3.el7.centos.x86_64.rpm ]; then
+    sudo yum install -y gcc-objc
+    rpm --define "_topdir $RPMBUILD" -ivh build/srpms/ragel-6.8-3.fc22.src.rpm
+    patch $RPMBUILD/SPECS/ragel.spec < dist/redhat/centos_dep/ragel.diff
+    rpmbuild --define "_topdir $RPMBUILD" -ba $RPMBUILD/SPECS/ragel.spec
+fi
+do_install ragel-6.8-3.el7.centos.x86_64.rpm

+if [ ! -f $RPMBUILD/RPMS/noarch/antlr3-tool-3.5.2-1.el7.centos.noarch.rpm ]; then
+   mkdir build/antlr3-tool-3.5.2
+   cp dist/redhat/centos_dep/antlr3 build/antlr3-tool-3.5.2
+   cd build/antlr3-tool-3.5.2
+   wget http://www.antlr3.org/download/antlr-3.5.2-complete-no-st3.jar
+   cd -
+   cd build
+   tar cJpf $RPMBUILD/SOURCES/antlr3-tool-3.5.2.tar.xz antlr3-tool-3.5.2
+   cd -
+   rpmbuild --define "_topdir $RPMBUILD" -ba dist/redhat/centos_dep/antlr3-tool.spec
+fi
+do_install antlr3-tool-3.5.2-1.el7.centos.noarch.rpm

+if [ ! -f $RPMBUILD/RPMS/x86_64/antlr3-C++-devel-3.5.2-1.el7.centos.x86_64.rpm ];then
+   wget -O build/3.5.2.tar.gz https://github.com/antlr/antlr3/archive/3.5.2.tar.gz
+   mv build/3.5.2.tar.gz $RPMBUILD/SOURCES
+   rpmbuild --define "_topdir $RPMBUILD" -ba dist/redhat/centos_dep/antlr3-C++-devel.spec
+fi
+do_install antlr3-C++-devel-3.5.2-1.el7.centos.x86_64.rpm
--- a/dist/redhat/scylla-server.spec.in
+++ b/dist/redhat/scylla-server.spec.in
@@ -13,9 +13,6 @@ BuildRequires:  libaio-devel boost-devel libstdc++-devel cryptopp-devel hwloc-de
 %{?rhel:BuildRequires: python34 devtoolset-3-gcc-c++}
 Requires:       libaio boost-program-options boost-system libstdc++ boost-thread cryptopp hwloc-libs numactl-libs libpciaccess libxml2 zlib thrift yaml-cpp lz4 snappy jsoncpp boost-filesystem systemd-libs xz-libs openssl-libs libcap libselinux libgcrypt libgpg-error elfutils-libs krb5-libs libcom_err libattr pcre elfutils-libelf bzip2-libs keyutils-libs

-# TODO: create our own bridge device for virtio
-Requires:       libvirt-daemon
-
 %description

 %prep
Author	SHA1	Message	Date
Takuya ASADA	4de587536c	dist: remove rpm dependency to libvirt This is for testing virtio mode, since we don't officially recommend to use virtio mode we should drop it. Signed-off-by: Takuya ASADA <syuu@cloudius-systems.com>	2015-09-24 12:02:36 +02:00
Asias He	49d6cba471	ami: Improve scylla raid setup Use all the disks except the one for rootfs for RAID0 which stores scylla data. If only one disk is available warn the user since currently our AMI's rootfs is not XFS. [fedora@ip-172-31-39-189 ~]$ cat WARN.TXT WARN: Scylla is not using XFS to store data. Performance will suffer. Tested on AWS with 1 disk, 2 disks, 7 disk case.	2015-09-24 01:20:04 -07:00
Avi Kivity	440b403089	Update scylla-ami submodule * dist/ami/files/scylla-ami c6ddbea...3f37184 (1): > Update reflector URL	2015-09-24 00:41:00 -07:00
Takuya ASADA	8587c4d6b3	dist: add scylla.repo to fetch scylla rpms on ami Mistakenly didn't included on yum repository for AMI patchset, but it's needed Signed-off-by: Takuya ASADA <syuu@cloudius-systems.com>	2015-09-23 23:38:42 -07:00
Asias He	5cb7ff0407	rpm: Improve rpm build scripts This makes we can build in a centos container.	2015-09-23 21:59:52 -07:00
Takuya ASADA	aaeccdee60	dist: update packages on ec2 instance first bootup Signed-off-by: Takuya ASADA <syuu@cloudius-systems.com>	2015-09-23 20:39:02 -07:00
Takuya ASADA	8b98fe5a1c	dist: use scylla repo on ami, instead of locally built rpms Signed-off-by: Takuya ASADA <syuu@cloudius-systems.com>	2015-09-23 20:39:02 -07:00
Takuya ASADA	2863b8098f	dist: move inline script to setup-ami.sh Signed-off-by: Takuya ASADA <syuu@cloudius-systems.com>	2015-09-23 20:39:01 -07:00
Tomasz Grabiec	74d8fe4067	db: Move "Populating Keyspace ..." message from WARN to INFO level WARN level is for messages which should draw log reader's attention, journalctl highlights them for example. Populating of keyspace is a fairly normal thing, so it should be logged on lower level.	2015-09-23 06:46:54 -07:00