mirror of
https://github.com/scylladb/scylladb.git
synced 2026-04-21 17:10:35 +00:00
b374f94b15
There are two reasons we may want NOT to use caching of pip deps: 1. When building a container, unless we specifically clean it up, it'll remain, even when we squash the image layers later. 2. When building a container, that cache is not useful, as we squash our containers later (so that layer is not cached really). And our CI cleans up the layers repo anyway. 3. Caching sometimes isn't great, and doesn't ensure we pick up the exact version (or latest) that we wish to... This PR changes two locations in Scylla, both of which (also) build containers, so certainly relevant for 1, 2 above and possibly 3. No real need to backport. Signed-off-by: Yaniv Kaul <yaniv.kaul@scylladb.com> Closes scylladb/scylladb#23822
427 lines
12 KiB
Bash
Executable File
427 lines
12 KiB
Bash
Executable File
#!/bin/bash -e
|
|
#
|
|
# This file is open source software, licensed to you under the terms
|
|
# of the Apache License, Version 2.0 (the "License"). See the NOTICE file
|
|
# distributed with this work for additional information regarding copyright
|
|
# ownership. You may not use this file except in compliance with the License.
|
|
#
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing,
|
|
# software distributed under the License is distributed on an
|
|
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
|
# KIND, either express or implied. See the License for the
|
|
# specific language governing permissions and limitations
|
|
# under the License.
|
|
#
|
|
|
|
trap 'echo "error $? in $0 line $LINENO"' ERR
|
|
|
|
# os-release may be missing in container environment by default.
|
|
if [ -f "/etc/os-release" ]; then
|
|
. /etc/os-release
|
|
elif [ -f "/etc/arch-release" ]; then
|
|
export ID=arch
|
|
else
|
|
echo "/etc/os-release missing."
|
|
exit 1
|
|
fi
|
|
|
|
debian_base_packages=(
|
|
clang
|
|
gdb
|
|
cargo
|
|
wabt
|
|
liblua5.3-dev
|
|
python3-aiohttp
|
|
python3-pyparsing
|
|
python3-colorama
|
|
python3-tabulate
|
|
libsnappy-dev
|
|
libjsoncpp-dev
|
|
rapidjson-dev
|
|
scylla-antlr35-c++-dev
|
|
git
|
|
pigz
|
|
libunistring-dev
|
|
libzstd-dev
|
|
libdeflate-dev
|
|
librapidxml-dev
|
|
libcrypto++-dev
|
|
libxxhash-dev
|
|
slapd
|
|
ldap-utils
|
|
libcpp-jwt-dev
|
|
)
|
|
|
|
fedora_packages=(
|
|
clang
|
|
compiler-rt
|
|
libasan
|
|
libubsan
|
|
gdb
|
|
lua-devel
|
|
yaml-cpp-devel
|
|
antlr3-tool
|
|
antlr3-C++-devel
|
|
jsoncpp-devel
|
|
rapidjson-devel
|
|
snappy-devel
|
|
libdeflate-devel
|
|
systemd-devel
|
|
cryptopp-devel
|
|
git
|
|
git-lfs
|
|
python
|
|
sudo
|
|
patchelf
|
|
python3
|
|
python3-aiohttp
|
|
python3-pip
|
|
python3-file-magic
|
|
python3-colorama
|
|
python3-tabulate
|
|
python3-boto3
|
|
python3-pytest
|
|
python3-pytest-asyncio
|
|
python3-redis
|
|
python3-unidiff
|
|
python3-humanfriendly
|
|
python3-jinja2
|
|
python3-deepdiff
|
|
python3-cryptography
|
|
dnf-utils
|
|
pigz
|
|
net-tools
|
|
tar
|
|
gzip
|
|
gawk
|
|
util-linux
|
|
ethtool
|
|
hwloc
|
|
glibc-langpack-en
|
|
xxhash-devel
|
|
makeself
|
|
libzstd-static libzstd-devel
|
|
lz4-static lz4-devel
|
|
rpm-build
|
|
devscripts
|
|
debhelper
|
|
fakeroot
|
|
file
|
|
dpkg-dev
|
|
curl
|
|
rust
|
|
cargo
|
|
rapidxml-devel
|
|
rust-std-static-wasm32-wasip1
|
|
wabt
|
|
binaryen
|
|
lcov
|
|
|
|
llvm-bolt
|
|
moreutils
|
|
iproute
|
|
llvm
|
|
openldap-servers
|
|
openldap-devel
|
|
toxiproxy
|
|
cyrus-sasl
|
|
fipscheck
|
|
cpp-jwt-devel
|
|
|
|
https://github.com/scylladb/cassandra-stress/releases/download/v3.17.5/cassandra-stress-3.17.5-1.noarch.rpm
|
|
)
|
|
|
|
# lld is not available on s390x, see
|
|
# https://src.fedoraproject.org/rpms/lld/c/aa6e69df60747496f8f22121ae8cc605c9d3498a?branch=rawhide
|
|
if [ "$(uname -m)" != "s390x" ]; then
|
|
fedora_packages+=(lld)
|
|
fi
|
|
|
|
fedora_python3_packages=(
|
|
python3-pyyaml
|
|
python3-urwid
|
|
python3-pyparsing
|
|
python3-requests
|
|
python3-setuptools
|
|
python3-psutil
|
|
python3-distro
|
|
python3-click
|
|
python3-six
|
|
python3-pyudev
|
|
)
|
|
|
|
# an associative array from packages to constrains
|
|
declare -A pip_packages=(
|
|
[scylla-driver]="==3.28.2" # lock-step with tools/cqlsh
|
|
[geomet]="<0.3,>=0.1"
|
|
[traceback-with-variables]=""
|
|
[scylla-api-client]=""
|
|
[treelib]=""
|
|
[allure-pytest]=""
|
|
[pytest-xdist]=""
|
|
[pykmip]=""
|
|
[universalasync]=""
|
|
[boto3-stubs[dynamodb]]=""
|
|
)
|
|
|
|
pip_symlinks=(
|
|
scylla-api-client
|
|
)
|
|
|
|
centos_packages=(
|
|
gdb
|
|
yaml-cpp-devel
|
|
scylla-antlr35-tool
|
|
scylla-antlr35-C++-devel
|
|
jsoncpp-devel snappy-devel
|
|
rapidjson-devel
|
|
scylla-boost163-static
|
|
scylla-python34-pyparsing20
|
|
systemd-devel
|
|
pigz
|
|
openldap-servers
|
|
openldap-devel
|
|
cpp-jwt-devel
|
|
)
|
|
|
|
# 1) glibc 2.30-3 has sys/sdt.h (systemtap include)
|
|
# some old containers may contain glibc older,
|
|
# so enforce update on that one.
|
|
# 2) if problems with signatures, ensure having fresh
|
|
# archlinux-keyring: pacman -Sy archlinux-keyring && pacman -Syyu
|
|
# 3) aur installations require having sudo and being
|
|
# a sudoer. makepkg does not work otherwise.
|
|
#
|
|
# aur: antlr3, antlr3-cpp-headers-git
|
|
arch_packages=(
|
|
gdb
|
|
base-devel
|
|
filesystem
|
|
git
|
|
glibc
|
|
jsoncpp
|
|
lua
|
|
python-pyparsing
|
|
python3
|
|
rapidjson
|
|
snappy
|
|
)
|
|
|
|
go_arch() {
|
|
local -A GO_ARCH=(
|
|
["x86_64"]=amd64
|
|
["aarch64"]=arm64
|
|
["s390x"]=s390x
|
|
)
|
|
echo ${GO_ARCH["$(arch)"]}
|
|
}
|
|
|
|
NODE_EXPORTER_VERSION=1.9.0
|
|
declare -A NODE_EXPORTER_CHECKSUM=(
|
|
["x86_64"]=e7b65ea30eec77180487d518081d3dcb121b975f6d95f1866dfb9156c5b24075
|
|
["aarch64"]=5314fae1efff19abf807cfc8bd7dadbd47a35565c1043c236ffb0689dc15ef4f
|
|
["s390x"]=089d2c2f87b4d716dd5ff006b89ab4424e7917f67830a8dd580d528f1d99ca58
|
|
)
|
|
NODE_EXPORTER_DIR=/opt/scylladb/dependencies
|
|
|
|
node_exporter_filename() {
|
|
echo "node_exporter-$NODE_EXPORTER_VERSION.linux-$(go_arch).tar.gz"
|
|
}
|
|
|
|
node_exporter_fullpath() {
|
|
echo "$NODE_EXPORTER_DIR/$(node_exporter_filename)"
|
|
}
|
|
|
|
node_exporter_checksum() {
|
|
sha256sum "$(node_exporter_fullpath)" | while read -r sum _; do [[ "$sum" == "${NODE_EXPORTER_CHECKSUM["$(arch)"]}" ]]; done
|
|
}
|
|
|
|
node_exporter_url() {
|
|
echo "https://github.com/prometheus/node_exporter/releases/download/v$NODE_EXPORTER_VERSION/$(node_exporter_filename)"
|
|
}
|
|
|
|
MINIO_BINARIES_DIR=/usr/local/bin
|
|
|
|
minio_server_url() {
|
|
echo "https://dl.minio.io/server/minio/release/linux-$(go_arch)/minio"
|
|
}
|
|
|
|
minio_client_url() {
|
|
echo "https://dl.min.io/client/mc/release/linux-$(go_arch)/mc"
|
|
}
|
|
|
|
minio_download_jobs() {
|
|
cfile=$(mktemp)
|
|
echo $(curl -s "$(minio_server_url).sha256sum" | cut -f1 -d' ') "${MINIO_BINARIES_DIR}/minio" > $cfile
|
|
echo $(curl -s "$(minio_client_url).sha256sum" | cut -f1 -d' ') "${MINIO_BINARIES_DIR}/mc" >> $cfile
|
|
sha256sum -c $cfile | grep -F FAILED | sed \
|
|
-e 's/:.*$//g' \
|
|
-e "s#${MINIO_BINARIES_DIR}/minio#$(minio_server_url) -o ${MINIO_BINARIES_DIR}/minio#" \
|
|
-e "s#${MINIO_BINARIES_DIR}/mc#$(minio_client_url) -o ${MINIO_BINARIES_DIR}/mc#"
|
|
rm -f ${cfile}
|
|
}
|
|
|
|
print_usage() {
|
|
echo "Usage: install-dependencies.sh [OPTION]..."
|
|
echo ""
|
|
echo " --print-python3-runtime-packages Print required python3 packages for Scylla"
|
|
echo " --print-pip-runtime-packages Print required pip packages for Scylla"
|
|
echo " --print-pip-symlinks Print list of pip provided commands which need to install to /usr/bin"
|
|
echo " --print-node-exporter-filename Print node_exporter filename"
|
|
exit 1
|
|
}
|
|
|
|
PRINT_PYTHON3=false
|
|
PRINT_PIP=false
|
|
PRINT_PIP_SYMLINK=false
|
|
PRINT_NODE_EXPORTER=false
|
|
while [ $# -gt 0 ]; do
|
|
case "$1" in
|
|
"--print-python3-runtime-packages")
|
|
PRINT_PYTHON3=true
|
|
shift 1
|
|
;;
|
|
"--print-pip-runtime-packages")
|
|
PRINT_PIP=true
|
|
shift 1
|
|
;;
|
|
"--print-pip-symlinks")
|
|
PRINT_PIP_SYMLINK=true
|
|
shift 1
|
|
;;
|
|
"--print-node-exporter-filename")
|
|
PRINT_NODE_EXPORTER=true
|
|
shift 1
|
|
;;
|
|
*)
|
|
print_usage
|
|
;;
|
|
esac
|
|
done
|
|
|
|
if $PRINT_PYTHON3; then
|
|
if [ "$ID" != "fedora" ]; then
|
|
echo "Unsupported Distribution: $ID"
|
|
exit 1
|
|
fi
|
|
echo "${fedora_python3_packages[@]}"
|
|
exit 0
|
|
fi
|
|
|
|
if $PRINT_PIP; then
|
|
echo "${!pip_packages[@]}"
|
|
exit 0
|
|
fi
|
|
|
|
if $PRINT_PIP_SYMLINK; then
|
|
echo "${pip_symlinks[@]}"
|
|
exit 0
|
|
fi
|
|
|
|
if $PRINT_NODE_EXPORTER; then
|
|
node_exporter_fullpath
|
|
exit 0
|
|
fi
|
|
|
|
umask 0022
|
|
|
|
./seastar/install-dependencies.sh
|
|
|
|
if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ]; then
|
|
apt-get -y install "${debian_base_packages[@]}"
|
|
if [ "$VERSION_ID" = "8" ]; then
|
|
apt-get -y install libsystemd-dev scylla-antlr35 libyaml-cpp-dev
|
|
elif [ "$VERSION_ID" = "14.04" ]; then
|
|
apt-get -y install scylla-antlr35 libyaml-cpp-dev
|
|
elif [ "$VERSION_ID" = "9" ]; then
|
|
apt-get -y install libsystemd-dev antlr3 scylla-libyaml-cpp05-dev
|
|
else
|
|
apt-get -y install libsystemd-dev antlr3 libyaml-cpp-dev
|
|
fi
|
|
apt-get -y install libssl-dev
|
|
|
|
echo -e "Configure example:\n\t./configure.py --enable-dpdk --mode=release --static-boost --static-yaml-cpp --compiler=/opt/scylladb/bin/g++-7 --cflags=\"-I/opt/scylladb/include -L/opt/scylladb/lib/x86-linux-gnu/\" --ldflags=\"-Wl,-rpath=/opt/scylladb/lib\""
|
|
elif [ "$ID" = "fedora" ]; then
|
|
fedora_packages+=(openssl-devel)
|
|
if rpm -q --quiet yum-utils; then
|
|
echo
|
|
echo "This script will install dnf-utils package, witch will conflict with currently installed package: yum-utils"
|
|
echo "Please remove the package and try to run this script again."
|
|
exit 1
|
|
fi
|
|
dnf install -y "${fedora_packages[@]}" "${fedora_python3_packages[@]}"
|
|
PIP_DEFAULT_ARGS="--only-binary=:all: -v"
|
|
pip_constrained_packages=""
|
|
for package in "${!pip_packages[@]}"
|
|
do
|
|
pip_constrained_packages="${pip_constrained_packages} ${package}${pip_packages[$package]}"
|
|
done
|
|
pip3 install --upgrade --no-cache-dir "$PIP_DEFAULT_ARGS" $pip_constrained_packages
|
|
|
|
if [ -f "$(node_exporter_fullpath)" ] && node_exporter_checksum; then
|
|
echo "$(node_exporter_filename) already exists, skipping download"
|
|
else
|
|
mkdir -p "$NODE_EXPORTER_DIR"
|
|
curl -fSL -o "$(node_exporter_fullpath)" "$(node_exporter_url)"
|
|
if ! node_exporter_checksum; then
|
|
echo "$(node_exporter_filename) download failed"
|
|
exit 1
|
|
fi
|
|
fi
|
|
elif [ "$ID" = "centos" ]; then
|
|
centos_packages+=(openssl-devel)
|
|
dnf install -y "${centos_packages[@]}"
|
|
echo -e "Configure example:\n\tpython3.4 ./configure.py --enable-dpdk --mode=release --static-boost --compiler=/opt/scylladb/bin/g++-7.3 --python python3.4 --ldflag=-Wl,-rpath=/opt/scylladb/lib64 --cflags=-I/opt/scylladb/include --with-antlr3=/opt/scylladb/bin/antlr3"
|
|
elif [ "$ID" == "arch" ]; then
|
|
# main
|
|
if [ "$EUID" -eq "0" ]; then
|
|
pacman -Sy --needed --noconfirm "${arch_packages[@]}"
|
|
else
|
|
echo "scylla: You now ran $0 as non-root. Run it again as root to execute the pacman part of the installation." 1>&2
|
|
fi
|
|
|
|
# aur
|
|
if [ ! -x /usr/bin/antlr3 ]; then
|
|
echo "Installing aur/antlr3..."
|
|
if (( EUID == 0 )); then
|
|
echo "You now ran $0 as root. This can only update dependencies with pacman. Please run again it as non-root to complete the AUR part of the installation." 1>&2
|
|
exit 1
|
|
fi
|
|
TEMP=$(mktemp -d)
|
|
pushd "$TEMP" > /dev/null || exit 1
|
|
git clone --depth 1 https://aur.archlinux.org/antlr3.git
|
|
cd antlr3 || exit 1
|
|
makepkg -si
|
|
popd > /dev/null || exit 1
|
|
fi
|
|
if [ ! -f /usr/include/antlr3.hpp ]; then
|
|
echo "Installing aur/antlr3-cpp-headers-git..."
|
|
if (( EUID == 0 )); then
|
|
echo "You now ran $0 as root. This can only update dependencies with pacman. Please run again it as non-root to complete the AUR part of the installation." 1>&2
|
|
exit 1
|
|
fi
|
|
TEMP=$(mktemp -d)
|
|
pushd "$TEMP" > /dev/null || exit 1
|
|
git clone --depth 1 https://aur.archlinux.org/antlr3-cpp-headers-git.git
|
|
cd antlr3-cpp-headers-git || exit 1
|
|
makepkg -si
|
|
popd > /dev/null || exit 1
|
|
fi
|
|
echo -e "Configure example:\n\t./configure.py\n\tninja release"
|
|
fi
|
|
|
|
cargo --config net.git-fetch-with-cli=true install cxxbridge-cmd --root /usr/local
|
|
|
|
CURL_ARGS=$(minio_download_jobs)
|
|
if [ ! -z "${CURL_ARGS}" ]; then
|
|
curl -fSL --remove-on-error --parallel --parallel-immediate ${CURL_ARGS}
|
|
chmod +x "${MINIO_BINARIES_DIR}/minio"
|
|
chmod +x "${MINIO_BINARIES_DIR}/mc"
|
|
else
|
|
echo "Minio server and client are up-to-date, skipping download"
|
|
fi
|