mirrors/scylladb
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
copilot/replace-clustering-range-usage
scylladb/install-dependencies.sh
Nadav Har'El 06dd3b2e64 install-dependencies.sh: add zlib 
Scylla uses zlib, through the header <zlib.h>, in sstable compression.
We also want to use it in Alternator for gzip-compressed requests.

We never actually required zlib explicltly in install-dependencies.sh,
we only get it through transitive dependencies. But it's better to
require it explicitly so this is what we do in this patch.

In Fedora, we use the newer, more efficient, zlib-ng which is API-
compatible with the classic zlib. Unfortunately, the Debian zlib-ng
package is *not* drop-in compatible with zlib (you need to include
a different header file <zlib-ng.h>) so we use the classic zlib.

Signed-off-by: Nadav Har'El <nyh@scylladb.com>

Closes scylladb/scylladb#27238
2025-12-03 19:30:36 +02:00

441 lines
12 KiB
Bash
Executable File
Raw Permalink Blame History

#!/bin/bash -e
#
# This file is open source software, licensed to you under the terms
# of the Apache License, Version 2.0 (the "License"). See the NOTICE file
# distributed with this work for additional information regarding copyright
# ownership. You may not use this file except in compliance with the License.
#
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
trap 'echo "error $? in $0 line $LINENO"' ERR
# os-release may be missing in container environment by default.
if [ -f "/etc/os-release" ]; then
. /etc/os-release
elif [ -f "/etc/arch-release" ]; then
export ID=arch
else
echo "/etc/os-release missing."
exit 1
fi
debian_base_packages=(
clang
gdb
cargo
wabt
liblua5.3-dev
python3-aiohttp
python3-pyparsing
python3-colorama
python3-tabulate
python3-pytest
python3-pytest-asyncio
python3-pytest-timeout
python3-pytest-sugar
libsnappy-dev
libjsoncpp-dev
rapidjson-dev
scylla-antlr35-c++-dev
git
pigz
libunistring-dev
libzstd-dev
libdeflate-dev
librapidxml-dev
libcrypto++-dev
libxxhash-dev
zlib1g-dev
slapd
ldap-utils
libcpp-jwt-dev
elfutils
curl
jq
git-lfs
e2fsprogs
fuse3
)
fedora_packages=(
clang
compiler-rt
libasan
libubsan
gdb
lua-devel
yaml-cpp-devel
antlr3-tool
antlr3-C++-devel
jsoncpp-devel
rapidjson-devel
snappy-devel
libdeflate-devel
systemd-devel
cryptopp-devel
git
git-lfs
python
sudo
patchelf
python3
python3-aiohttp
python3-pip
python3-file-magic
python3-colorama
python3-tabulate
python3-boto3
python3-pytest
python3-pytest-asyncio
python3-pytest-timeout
python3-pytest-sugar
python3-unidiff
python3-humanfriendly
python3-jinja2
python3-deepdiff
python3-cryptography
dnf-utils
pigz
net-tools
tar
gzip
gawk
util-linux
ethtool
hwloc
glibc-langpack-en
xxhash-devel
makeself
libzstd-static libzstd-devel
lz4-static lz4-devel
zlib-ng-compat-devel
rpm-build
devscripts
debhelper
fakeroot
file
dpkg-dev
curl
rust
cargo
rapidxml-devel
rust-std-static-wasm32-wasip1
wabt
binaryen
lcov
e2fsprogs
fuse3
lld
llvm-bolt
moreutils
iproute
llvm
openldap-servers
openldap-devel
toxiproxy
cyrus-sasl
fipscheck
cpp-jwt-devel
podman
buildah
https://github.com/scylladb/cassandra-stress/releases/download/v3.18.1/cassandra-stress-java21-3.18.1-1.noarch.rpm
elfutils
jq
)
fedora_python3_packages=(
python3-pyyaml
python3-urwid
python3-pyparsing
python3-requests
python3-setuptools
python3-psutil
python3-distro
python3-click
python3-six
python3-pyudev
)
# an associative array from packages to constrains
declare -A pip_packages=(
[scylla-driver]="==$(cat tools/cqlsh/requirements.txt | grep scylla-driver | cut -d= -f3)"
[geomet]=""
[traceback-with-variables]=""
[scylla-api-client]=""
[treelib]=""
[allure-pytest]=""
[pytest-xdist]=""
[pykmip]=""
[universalasync]=""
[boto3-stubs[dynamodb]]=""
[setuptools_scm]=""
)
pip_symlinks=(
scylla-api-client
)
centos_packages=(
gdb
yaml-cpp-devel
scylla-antlr35-tool
scylla-antlr35-C++-devel
jsoncpp-devel snappy-devel
rapidjson-devel
scylla-boost163-static
scylla-python34-pyparsing20
systemd-devel
pigz
openldap-servers
openldap-devel
cpp-jwt-devel
)
# 1) glibc 2.30-3 has sys/sdt.h (systemtap include)
# some old containers may contain glibc older,
# so enforce update on that one.
# 2) if problems with signatures, ensure having fresh
# archlinux-keyring: pacman -Sy archlinux-keyring && pacman -Syyu
# 3) aur installations require having sudo and being
# a sudoer. makepkg does not work otherwise.
#
# aur: antlr3, antlr3-cpp-headers-git
arch_packages=(
gdb
base-devel
filesystem
git
glibc
jsoncpp
lua
python-pyparsing
python3
rapidjson
snappy
)
go_arch() {
local -A GO_ARCH=(
["x86_64"]=amd64
["aarch64"]=arm64
)
echo ${GO_ARCH["$(arch)"]}
}
NODE_EXPORTER_VERSION=1.10.2
declare -A NODE_EXPORTER_CHECKSUM=(
["x86_64"]=c46e5b6f53948477ff3a19d97c58307394a29fe64a01905646f026ddc32cb65b
["aarch64"]=de69ec8341c8068b7c8e4cfe3eb85065d24d984a3b33007f575d307d13eb89a6
)
NODE_EXPORTER_DIR=/opt/scylladb/dependencies
node_exporter_filename() {
echo "node_exporter-$NODE_EXPORTER_VERSION.linux-$(go_arch).tar.gz"
}
node_exporter_fullpath() {
echo "$NODE_EXPORTER_DIR/$(node_exporter_filename)"
}
node_exporter_checksum() {
sha256sum "$(node_exporter_fullpath)" | while read -r sum _; do [[ "$sum" == "${NODE_EXPORTER_CHECKSUM["$(arch)"]}" ]]; done
}
node_exporter_url() {
echo "https://github.com/prometheus/node_exporter/releases/download/v$NODE_EXPORTER_VERSION/$(node_exporter_filename)"
}
MINIO_BINARIES_DIR=/usr/local/bin
minio_server_url() {
echo "https://dl.minio.io/server/minio/release/linux-$(go_arch)/minio"
}
minio_client_url() {
echo "https://dl.min.io/client/mc/release/linux-$(go_arch)/mc"
}
minio_download_jobs() {
cfile=$(mktemp)
echo $(curl -s "$(minio_server_url).sha256sum" | cut -f1 -d' ') "${MINIO_BINARIES_DIR}/minio" > $cfile
echo $(curl -s "$(minio_client_url).sha256sum" | cut -f1 -d' ') "${MINIO_BINARIES_DIR}/mc" >> $cfile
sha256sum -c $cfile | grep -F FAILED | sed \
-e 's/:.*$//g' \
-e "s#${MINIO_BINARIES_DIR}/minio#$(minio_server_url) -o ${MINIO_BINARIES_DIR}/minio#" \
-e "s#${MINIO_BINARIES_DIR}/mc#$(minio_client_url) -o ${MINIO_BINARIES_DIR}/mc#"
rm -f ${cfile}
}
print_usage() {
echo "Usage: install-dependencies.sh [OPTION]..."
echo ""
echo " --print-python3-runtime-packages Print required python3 packages for Scylla"
echo " --print-pip-runtime-packages Print required pip packages for Scylla"
echo " --print-pip-symlinks Print list of pip provided commands which need to install to /usr/bin"
echo " --print-node-exporter-filename Print node_exporter filename"
exit 1
}
PRINT_PYTHON3=false
PRINT_PIP=false
PRINT_PIP_SYMLINK=false
PRINT_NODE_EXPORTER=false
while [ $# -gt 0 ]; do
case "$1" in
"--print-python3-runtime-packages")
PRINT_PYTHON3=true
shift 1
;;
"--print-pip-runtime-packages")
PRINT_PIP=true
shift 1
;;
"--print-pip-symlinks")
PRINT_PIP_SYMLINK=true
shift 1
;;
"--print-node-exporter-filename")
PRINT_NODE_EXPORTER=true
shift 1
;;
*)
print_usage
;;
esac
done
if $PRINT_PYTHON3; then
if [ "$ID" != "fedora" ]; then
echo "Unsupported Distribution: $ID"
exit 1
fi
echo "${fedora_python3_packages[@]}"
exit 0
fi
if $PRINT_PIP; then
echo "${!pip_packages[@]}"
exit 0
fi
if $PRINT_PIP_SYMLINK; then
echo "${pip_symlinks[@]}"
exit 0
fi
if $PRINT_NODE_EXPORTER; then
node_exporter_fullpath
exit 0
fi
umask 0022
./seastar/install-dependencies.sh
if [ "$ID" = "ubuntu" ] || [ "$ID" = "debian" ]; then
apt-get -y install "${debian_base_packages[@]}"
if [ "$VERSION_ID" = "8" ]; then
apt-get -y install libsystemd-dev scylla-antlr35 libyaml-cpp-dev
elif [ "$VERSION_ID" = "14.04" ]; then
apt-get -y install scylla-antlr35 libyaml-cpp-dev
elif [ "$VERSION_ID" = "9" ]; then
apt-get -y install libsystemd-dev antlr3 scylla-libyaml-cpp05-dev
else
apt-get -y install libsystemd-dev antlr3 libyaml-cpp-dev
fi
apt-get -y install libssl-dev
echo -e "Configure example:\n\t./configure.py --enable-dpdk --mode=release --static-boost --static-yaml-cpp --compiler=/opt/scylladb/bin/g++-7 --cflags=\"-I/opt/scylladb/include -L/opt/scylladb/lib/x86-linux-gnu/\" --ldflags=\"-Wl,-rpath=/opt/scylladb/lib\""
elif [ "$ID" = "fedora" ]; then
fedora_packages+=(openssl-devel)
if rpm -q --quiet yum-utils; then
echo
echo "This script will install dnf-utils package, witch will conflict with currently installed package: yum-utils"
echo "Please remove the package and try to run this script again."
exit 1
fi
dnf install -y "${fedora_packages[@]}" "${fedora_python3_packages[@]}"
PIP_DEFAULT_ARGS="--only-binary=:all: -v"
pip_constrained_packages=""
for package in "${!pip_packages[@]}"
do
pip_constrained_packages="${pip_constrained_packages} ${package}${pip_packages[$package]}"
done
pip3 install --upgrade --no-cache-dir "$PIP_DEFAULT_ARGS" $pip_constrained_packages
if [ -f "$(node_exporter_fullpath)" ] && node_exporter_checksum; then
echo "$(node_exporter_filename) already exists, skipping download"
else
mkdir -p "$NODE_EXPORTER_DIR"
curl -fSL -o "$(node_exporter_fullpath)" "$(node_exporter_url)"
if ! node_exporter_checksum; then
echo "$(node_exporter_filename) download failed"
exit 1
fi
fi
elif [ "$ID" = "centos" ]; then
centos_packages+=(openssl-devel)
dnf install -y "${centos_packages[@]}"
echo -e "Configure example:\n\tpython3.4 ./configure.py --enable-dpdk --mode=release --static-boost --compiler=/opt/scylladb/bin/g++-7.3 --python python3.4 --ldflag=-Wl,-rpath=/opt/scylladb/lib64 --cflags=-I/opt/scylladb/include --with-antlr3=/opt/scylladb/bin/antlr3"
elif [ "$ID" == "arch" ]; then
# main
if [ "$EUID" -eq "0" ]; then
pacman -Sy --needed --noconfirm "${arch_packages[@]}"
else
echo "scylla: You now ran $0 as non-root. Run it again as root to execute the pacman part of the installation." 1>&2
fi
# aur
if [ ! -x /usr/bin/antlr3 ]; then
echo "Installing aur/antlr3..."
if (( EUID == 0 )); then
echo "You now ran $0 as root. This can only update dependencies with pacman. Please run again it as non-root to complete the AUR part of the installation." 1>&2
exit 1
fi
TEMP=$(mktemp -d)
pushd "$TEMP" > /dev/null || exit 1
git clone --depth 1 https://aur.archlinux.org/antlr3.git
cd antlr3 || exit 1
makepkg -si
popd > /dev/null || exit 1
fi
if [ ! -f /usr/include/antlr3.hpp ]; then
echo "Installing aur/antlr3-cpp-headers-git..."
if (( EUID == 0 )); then
echo "You now ran $0 as root. This can only update dependencies with pacman. Please run again it as non-root to complete the AUR part of the installation." 1>&2
exit 1
fi
TEMP=$(mktemp -d)
pushd "$TEMP" > /dev/null || exit 1
git clone --depth 1 https://aur.archlinux.org/antlr3-cpp-headers-git.git
cd antlr3-cpp-headers-git || exit 1
makepkg -si
popd > /dev/null || exit 1
fi
echo -e "Configure example:\n\t./configure.py\n\tninja release"
fi
cargo --config net.git-fetch-with-cli=true install cxxbridge-cmd --version 1.0.83 --root /usr/local
CURL_ARGS=$(minio_download_jobs)
if [ ! -z "${CURL_ARGS}" ]; then
curl -fSL --remove-on-error --parallel --parallel-immediate ${CURL_ARGS}
chmod +x "${MINIO_BINARIES_DIR}/minio"
chmod +x "${MINIO_BINARIES_DIR}/mc"
else
echo "Minio server and client are up-to-date, skipping download"
fi
Reference in New Issue View Git Blame Copy Permalink